diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 1b3a16eebd2c..b5cf5bb4dc80 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -5,51 +5,41 @@ labels: "bug" --- **Bug Report** (A clear and concise description of what the bug is.) **To Reproduce** -(Write your steps here:) - -1. Step 1... -2. Step 2... -3. Step 3... +```python +# Ideally, a small sample program that demonstrates the problem. +# Or even better, a reproducible playground link https://mypy-play.net/ (use the "Gist" button) +``` **Expected Behavior** -(Write what you thought would happen.) - **Actual Behavior** - - -(Write what happened.) + **Your Environment** @@ -59,9 +49,5 @@ for this report: https://github.com/python/typeshed/issues - Mypy command-line flags: - Mypy configuration options from `mypy.ini` (and other config files): - Python version used: -- Operating system and version: - + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4794ec05c906..696eb8aee125 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,22 +1,12 @@ -### Have you read the [Contributing Guidelines](https://github.com/python/mypy/blob/master/CONTRIBUTING.md)? - -(Once you have, delete this section. If you leave it in, your PR may be closed without action.) - -### Description - - + (Explain how this PR changes mypy.) -## Test Plan - - -(Write your test plan here. If you changed any code, please provide us with clear instructions on how you verified your changes work.) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 59ee859f1414..d26372aa6635 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -13,9 +13,12 @@ on: - 'mypy/stubgen.py' - 'mypy/stubgenc.py' - 'mypy/test/**' - - 'scripts/**' - 'test-data/**' +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: mypy_primer: name: Run mypy_primer @@ -24,7 +27,7 @@ jobs: contents: read strategy: matrix: - shard-index: [0, 1, 2] + shard-index: [0, 1, 2, 3, 4] fail-fast: false steps: - uses: actions/checkout@v3 @@ -57,7 +60,7 @@ jobs: mypy_primer \ --repo mypy_to_test \ --new $GITHUB_SHA --old base_commit \ - --num-shards 3 --shard-index ${{ matrix.shard-index }} \ + --num-shards 5 --shard-index ${{ matrix.shard-index }} \ --debug \ --output concise \ | tee diff_${{ matrix.shard-index }}.txt diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml index 94d387fb7da0..2056fc5a40c0 100644 --- a/.github/workflows/mypy_primer_comment.yml +++ b/.github/workflows/mypy_primer_comment.yml @@ -15,6 +15,7 @@ jobs: comment: name: Comment PR from mypy_primer runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Download diffs uses: actions/github-script@v6 diff --git a/.github/workflows/sync_typeshed.yml b/.github/workflows/sync_typeshed.yml new file mode 100644 index 000000000000..1db2e846f099 --- /dev/null +++ b/.github/workflows/sync_typeshed.yml @@ -0,0 +1,33 @@ +name: Sync typeshed + +on: + workflow_dispatch: + schedule: + - cron: "0 0 1,15 * *" + +permissions: + contents: write + pull-requests: write + +jobs: + sync_typeshed: + name: Sync typeshed + if: github.repository == 'python/mypy' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + # TODO: use whatever solution ends up working for + # https://github.com/python/typeshed/issues/8434 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: git config + run: | + git config --global user.name mypybot + git config --global user.email '<>' + - name: Sync typeshed + run: | + python -m pip install requests==2.28.1 + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python misc/sync-typeshed.py --make-pr diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e8f8a2a05e2b..3fdb83ff15b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ on: - CREDITS - LICENSE -concurrency: +concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -26,11 +26,6 @@ jobs: fail-fast: false matrix: include: - - name: Test suite with py37-windows-32 - python: '3.7' - arch: x86 - os: windows-latest - toxenv: py37 - name: Test suite with py37-windows-64 python: '3.7' arch: x64 @@ -68,6 +63,13 @@ jobs: os: ubuntu-latest toxenv: py tox_extra_args: "-n 2" + - name: Test suite with py311-ubuntu, mypyc-compiled + python: '3.11' + arch: x64 + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 2" + test_mypyc: true - name: mypyc runtime tests with py37-macos python: '3.7' arch: x64 @@ -120,26 +122,8 @@ jobs: if: ${{ matrix.test_mypyc }} run: | pip install -r test-requirements.txt - CC=clang MYPYC_OPT_LEVEL=0 python3 setup.py --use-mypyc build_ext --inplace + CC=clang MYPYC_OPT_LEVEL=0 MYPY_USE_MYPYC=1 pip install -e . - name: Setup tox environment run: tox -e ${{ matrix.toxenv }} --notest - name: Test run: tox -e ${{ matrix.toxenv }} --skip-pkg-install -- ${{ matrix.tox_extra_args }} - - python-nightly: - runs-on: ubuntu-latest - name: Test suite with Python nightly - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.11-dev' - - name: Install tox - run: pip install --upgrade 'setuptools!=50' tox==3.24.5 - - name: Setup tox environment - run: tox -e py --notest - - name: Test - run: tox -e py --skip-pkg-install -- "-n 2" - continue-on-error: true - - name: Mark as a success - run: exit 0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 53a8c5b541db..af1bb320be6d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: 22.6.0 # must match test-requirements.txt + rev: 22.10.0 # must match test-requirements.txt hooks: - id: black - repo: https://github.com/pycqa/isort @@ -8,9 +8,9 @@ repos: hooks: - id: isort - repo: https://github.com/pycqa/flake8 - rev: 3.9.2 # must match test-requirements.txt + rev: 5.0.4 # must match test-requirements.txt hooks: - id: flake8 additional_dependencies: - - flake8-bugbear==22.7.1 # must match test-requirements.txt - - flake8-noqa==1.2.8 # must match test-requirements.txt + - flake8-bugbear==22.9.23 # must match test-requirements.txt + - flake8-noqa==1.2.9 # must match test-requirements.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c433eaee05b9..193c9f27c85b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,10 +54,16 @@ python3 runtests.py You can also use `tox` to run tests (`tox` handles setting up the test environment for you): ``` tox -e py + +# Or some specific python version: +tox -e py39 + +# Or some specific command: +tox -e lint ``` Some useful commands for running specific tests include: -``` +```bash # Use mypy to check mypy's own code python3 runtests.py self # or equivalently: diff --git a/README.md b/README.md index 68e0975a791b..95cacb05d682 100644 --- a/README.md +++ b/README.md @@ -58,10 +58,6 @@ Python is a dynamic language, so usually you'll only see errors in your code when you attempt to run it. Mypy is a *static* checker, so it finds bugs in your programs without even running them! -Mypy is designed with gradual typing in mind. This means you can add type -hints to your code base slowly and that you can always fall back to dynamic -typing when static typing is not convenient. - Here is a small example to whet your appetite: ```python @@ -69,13 +65,26 @@ number = input("What is your favourite number?") print("It is", number + 1) # error: Unsupported operand types for + ("str" and "int") ``` -See [the documentation](https://mypy.readthedocs.io/en/stable/index.html) for more examples. +Adding type hints for mypy does not interfere with the way your program would +otherwise run. Think of type hints as similar to comments! You can always use +the Python interpreter to run your code, even if mypy reports errors. + +Mypy is designed with gradual typing in mind. This means you can add type +hints to your code base slowly and that you can always fall back to dynamic +typing when static typing is not convenient. + +Mypy has a powerful and easy-to-use type system, supporting features such as +type inference, generics, callable types, tuple types, union types, +structural subtyping and more. Using mypy will make your programs easier to +understand, debug, and maintain. + +See [the documentation](https://mypy.readthedocs.io/en/stable/index.html) for +more examples and information. In particular, see: - [type hints cheat sheet](https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html) - [getting started](https://mypy.readthedocs.io/en/stable/getting_started.html) - Quick start ----------- diff --git a/build-requirements.txt b/build-requirements.txt index dabc9b14c493..52c518d53bc2 100644 --- a/build-requirements.txt +++ b/build-requirements.txt @@ -1,3 +1,5 @@ +# NOTE: this needs to be kept in sync with the "requires" list in pyproject.toml -r mypy-requirements.txt +types-psutil types-setuptools -types-typed-ast>=1.5.0,<1.6.0 +types-typed-ast>=1.5.8,<1.6.0 diff --git a/docs/source/builtin_types.rst b/docs/source/builtin_types.rst index 7ff9bd3c38e9..37b56169d879 100644 --- a/docs/source/builtin_types.rst +++ b/docs/source/builtin_types.rst @@ -15,8 +15,8 @@ Type Description ``int`` integer ``float`` floating point number ``bool`` boolean value (subclass of ``int``) -``str`` string (unicode in Python 3) -``bytes`` 8-bit string +``str`` text, sequence of unicode codepoints +``bytes`` 8-bit string, sequence of byte values ``object`` an arbitrary object (``object`` is the common base class) ====================== =============================== diff --git a/docs/source/cheat_sheet_py3.rst b/docs/source/cheat_sheet_py3.rst index 29a25f38eac2..7179318e31b8 100644 --- a/docs/source/cheat_sheet_py3.rst +++ b/docs/source/cheat_sheet_py3.rst @@ -1,34 +1,27 @@ .. _cheat-sheet-py3: -Type hints cheat sheet (Python 3) -================================= - -This document is a quick cheat sheet showing how the :pep:`484` type -annotation notation represents various common types in Python 3. - -.. note:: - - Technically many of the type annotations shown below are redundant, - because mypy can derive them from the type of the expression. So - many of the examples have a dual purpose: show how to write the - annotation, and show the inferred types. +Type hints cheat sheet +====================== +This document is a quick cheat sheet showing how to use type +annotations for various common types in Python. Variables ********* -Python 3.6 introduced a syntax for annotating variables in :pep:`526` -and we use it in most examples. +Technically many of the type annotations shown below are redundant, +since mypy can usually infer the type of a variable from its value. +See :ref:`type-inference-and-annotations` for more details. .. code-block:: python - # This is how you declare the type of a variable type in Python 3.6 + # This is how you declare the type of a variable age: int = 1 # You don't need to initialize a variable to annotate it a: int # Ok (no value at runtime until assigned) - # The latter is useful in conditional branches + # Doing so is useful in conditional branches child: bool if age < 18: child = True @@ -36,45 +29,50 @@ and we use it in most examples. child = False -Built-in types -************** +Useful built-in types +********************* .. code-block:: python - - from typing import List, Set, Dict, Tuple, Optional - - # For simple built-in types, just use the name of the type + # For most types, just use the name of the type x: int = 1 x: float = 1.0 x: bool = True x: str = "test" x: bytes = b"test" - # For collections, the type of the collection item is in brackets - # (Python 3.9+) + # For collections on Python 3.9+, the type of the collection item is in brackets x: list[int] = [1] x: set[int] = {6, 7} - # In Python 3.8 and earlier, the name of the collection type is - # capitalized, and the type is imported from the 'typing' module - x: List[int] = [1] - x: Set[int] = {6, 7} - # For mappings, we need the types of both keys and values x: dict[str, float] = {"field": 2.0} # Python 3.9+ - x: Dict[str, float] = {"field": 2.0} # For tuples of fixed size, we specify the types of all the elements x: tuple[int, str, float] = (3, "yes", 7.5) # Python 3.9+ - x: Tuple[int, str, float] = (3, "yes", 7.5) # For tuples of variable size, we use one type and ellipsis x: tuple[int, ...] = (1, 2, 3) # Python 3.9+ + + # On Python 3.8 and earlier, the name of the collection type is + # capitalized, and the type is imported from the 'typing' module + from typing import List, Set, Dict, Tuple + x: List[int] = [1] + x: Set[int] = {6, 7} + x: Dict[str, float] = {"field": 2.0} + x: Tuple[int, str, float] = (3, "yes", 7.5) x: Tuple[int, ...] = (1, 2, 3) - # Use Optional[] for values that could be None - x: Optional[str] = some_function() + from typing import Union, Optional + + # On Python 3.10+, use the | operator when something could be one of a few types + x: list[int | str] = [3, 5, "test", "fun"] # Python 3.10+ + # On earlier versions, use Union + x: list[Union[int, str]] = [3, 5, "test", "fun"] + + # Use Optional[X] for a value that could be None + # Optional[X] is the same as X | None or Union[X, None] + x: Optional[str] = "something" if some_condition() else None # Mypy understands a value can't be None in an if-statement if x is not None: print(x.upper()) @@ -85,8 +83,6 @@ Built-in types Functions ********* -Python 3 supports an annotation syntax for function declarations. - .. code-block:: python from typing import Callable, Iterator, Union, Optional @@ -99,9 +95,10 @@ Python 3 supports an annotation syntax for function declarations. def plus(num1: int, num2: int) -> int: return num1 + num2 - # Add default value for an argument after the type annotation - def f(num1: int, my_float: float = 3.5) -> float: - return num1 + my_float + # If a function does not return a value, use None as the return type + # Default value for an argument goes after the type annotation + def show(value: str, excitement: int = 10) -> None: + print(value + "!" * excitement) # This is how you annotate a callable (function) value x: Callable[[int, float], float] = f @@ -119,78 +116,121 @@ Python 3 supports an annotation syntax for function declarations. sender: str, cc: Optional[list[str]], bcc: Optional[list[str]], - subject='', + subject: str = '', body: Optional[list[str]] = None ) -> bool: ... - # An argument can be declared positional-only by giving it a name - # starting with two underscores: - def quux(__x: int) -> None: + # Mypy understands positional-only and keyword-only arguments + # Positional-only arguments can also be marked by using a name starting with + # two underscores + def quux(x: int, / *, y: int) -> None: pass - quux(3) # Fine - quux(__x=3) # Error + quux(3, y=5) # Ok + quux(3, 5) # error: Too many positional arguments for "quux" + quux(x=3, y=5) # error: Unexpected keyword argument "x" for "quux" + + # This says each positional arg and each keyword arg is a "str" + def call(self, *args: str, **kwargs: str) -> str: + reveal_type(args) # Revealed type is "tuple[str, ...]" + reveal_type(kwargs) # Revealed type is "dict[str, str]" + request = make_request(*args, **kwargs) + return self.do_api_query(request) + +Classes +******* + +.. code-block:: python + + class MyClass: + # You can optionally declare instance variables in the class body + attr: int + # This is an instance variable with a default value + charge_percent: int = 100 + + # The "__init__" method doesn't return anything, so it gets return + # type "None" just like any other method that doesn't return anything + def __init__(self) -> None: + ... + + # For instance methods, omit type for "self" + def my_method(self, num: int, str1: str) -> str: + return num * str1 + + # User-defined classes are valid as types in annotations + x: MyClass = MyClass() + + # You can also declare the type of an attribute in "__init__" + class Box: + def __init__(self) -> None: + self.items: list[str] = [] + + # You can use the ClassVar annotation to declare a class variable + class Car: + seats: ClassVar[int] = 4 + passengers: ClassVar[list[str]] + + # If you want dynamic attributes on your class, have it + # override "__setattr__" or "__getattr__": + # - "__getattr__" allows for dynamic access to names + # - "__setattr__" allows for dynamic assignment to names + class A: + # This will allow assignment to any A.x, if x is the same type as "value" + # (use "value: Any" to allow arbitrary types) + def __setattr__(self, name: str, value: int) -> None: ... + + # This will allow access to any A.x, if x is compatible with the return type + def __getattr__(self, name: str) -> int: ... + + a.foo = 42 # Works + a.bar = 'Ex-parrot' # Fails type checking When you're puzzled or when things are complicated ************************************************** .. code-block:: python - from typing import Union, Any, Optional, cast + from typing import Union, Any, Optional, TYPE_CHECKING, cast # To find out what type mypy infers for an expression anywhere in # your program, wrap it in reveal_type(). Mypy will print an error # message with the type; remove it again before running the code. - reveal_type(1) # -> Revealed type is "builtins.int" - - # Use Union when something could be one of a few types - x: list[Union[int, str]] = [3, 5, "test", "fun"] - - # Use Any if you don't know the type of something or it's too - # dynamic to write a type for - x: Any = mystery_function() + reveal_type(1) # Revealed type is "builtins.int" # If you initialize a variable with an empty container or "None" - # you may have to help mypy a bit by providing a type annotation + # you may have to help mypy a bit by providing an explicit type annotation x: list[str] = [] x: Optional[str] = None - # This makes each positional arg and each keyword arg a "str" - def call(self, *args: str, **kwargs: str) -> str: - request = make_request(*args, **kwargs) - return self.do_api_query(request) + # Use Any if you don't know the type of something or it's too + # dynamic to write a type for + x: Any = mystery_function() + # Mypy will let you do anything with x! + x.whatever() * x["you"] + x("want") - any(x) and all(x) is super # no errors # Use a "type: ignore" comment to suppress errors on a given line, # when your code confuses mypy or runs into an outright bug in mypy. - # Good practice is to comment every "ignore" with a bug link - # (in mypy, typeshed, or your own code) or an explanation of the issue. - x = confusing_function() # type: ignore # https://github.com/python/mypy/issues/1167 + # Good practice is to add a comment explaining the issue. + x = confusing_function() # type: ignore # confusing_function won't return None here because ... # "cast" is a helper function that lets you override the inferred # type of an expression. It's only for mypy -- there's no runtime check. a = [4] b = cast(list[int], a) # Passes fine - c = cast(list[str], a) # Passes fine (no runtime check) - reveal_type(c) # -> Revealed type is "builtins.list[builtins.str]" - print(c) # -> [4]; the object is not cast - - # If you want dynamic attributes on your class, have it override "__setattr__" - # or "__getattr__" in a stub or in your source code. - # - # "__setattr__" allows for dynamic assignment to names - # "__getattr__" allows for dynamic access to names - class A: - # This will allow assignment to any A.x, if x is the same type as "value" - # (use "value: Any" to allow arbitrary types) - def __setattr__(self, name: str, value: int) -> None: ... - - # This will allow access to any A.x, if x is compatible with the return type - def __getattr__(self, name: str) -> int: ... - - a.foo = 42 # Works - a.bar = 'Ex-parrot' # Fails type checking + c = cast(list[str], a) # Passes fine despite being a lie (no runtime check) + reveal_type(c) # Revealed type is "builtins.list[builtins.str]" + print(c) # Still prints [4] ... the object is not changed or casted at runtime + + # Use "TYPE_CHECKING" if you want to have code that mypy can see but will not + # be executed at runtime (or to have code that mypy can't see) + if TYPE_CHECKING: + import json + else: + import orjson as json # mypy is unaware of this +In some cases type annotations can cause issues at runtime, see +:ref:`runtime_troubles` for dealing with this. Standard "duck types" ********************* @@ -216,7 +256,7 @@ that are common in idiomatic Python are standardized. # Mapping describes a dict-like object (with "__getitem__") that we won't # mutate, and MutableMapping one (with "__setitem__") that we might def f(my_mapping: Mapping[int, str]) -> list[int]: - my_mapping[5] = 'maybe' # if we try this, mypy will throw an error... + my_mapping[5] = 'maybe' # mypy will complain about this line... return list(my_mapping.keys()) f({3: 'yes', 4: 'no'}) @@ -230,40 +270,6 @@ that are common in idiomatic Python are standardized. You can even make your own duck types using :ref:`protocol-types`. -Classes -******* - -.. code-block:: python - - class MyClass: - # You can optionally declare instance variables in the class body - attr: int - # This is an instance variable with a default value - charge_percent: int = 100 - - # The "__init__" method doesn't return anything, so it gets return - # type "None" just like any other method that doesn't return anything - def __init__(self) -> None: - ... - - # For instance methods, omit type for "self" - def my_method(self, num: int, str1: str) -> str: - return num * str1 - - # User-defined classes are valid as types in annotations - x: MyClass = MyClass() - - # You can use the ClassVar annotation to declare a class variable - class Car: - seats: ClassVar[int] = 4 - passengers: ClassVar[list[str]] - - # You can also declare the type of an attribute in "__init__" - class Box: - def __init__(self) -> None: - self.items: list[str] = [] - - Coroutines and asyncio ********************** @@ -288,11 +294,7 @@ Miscellaneous .. code-block:: python import sys - import re - from typing import Match, IO - - # "typing.Match" describes regex matches from the re module - x: Match[str] = re.match(r'[0-9]+', "15") + from typing import IO # Use IO[] for functions that should accept or return any # object that comes from an open() call (IO[] does not @@ -307,7 +309,7 @@ Miscellaneous # Forward references are useful if you want to reference a class before # it is defined - def f(foo: A) -> int: # This will fail + def f(foo: A) -> int: # This will fail at runtime with 'A' is not defined ... class A: diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 1eaba59a10c2..1d4164192318 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -308,6 +308,26 @@ however: in this case, but any attempt to construct an instance will be flagged as an error. +Mypy allows you to omit the body for an abstract method, but if you do so, +it is unsafe to call such method via ``super()``. For example: + +.. code-block:: python + + from abc import abstractmethod + class Base: + @abstractmethod + def foo(self) -> int: pass + @abstractmethod + def bar(self) -> int: + return 0 + class Sub(Base): + def foo(self) -> int: + return super().foo() + 1 # error: Call to abstract method "foo" of "Base" + # with trivial body via super() is unsafe + @abstractmethod + def bar(self) -> int: + return super().bar() + 1 # This is OK however. + A class can inherit any number of classes, both abstract and concrete. As with normal overrides, a dynamically typed method can override or implement a statically typed method defined in any base diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index e2175a7f35d4..83d2983472be 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -129,30 +129,12 @@ Import discovery The following flags customize how exactly mypy discovers and follows imports. -.. option:: --namespace-packages - - This flag enables import discovery to use namespace packages (see - :pep:`420`). In particular, this allows discovery of imported - packages that don't have an ``__init__.py`` (or ``__init__.pyi``) - file. - - Namespace packages are found (using the PEP 420 rules, which - prefers "classic" packages over namespace packages) along the - module search path -- this is primarily set from the source files - passed on the command line, the ``MYPYPATH`` environment variable, - and the :confval:`mypy_path` config option. - - This flag affects how mypy finds modules and packages explicitly passed on - the command line. It also affects how mypy determines fully qualified module - names for files passed on the command line. See :ref:`Mapping file paths to - modules ` for details. - .. option:: --explicit-package-bases This flag tells mypy that top-level packages will be based in either the current directory, or a member of the ``MYPYPATH`` environment variable or :confval:`mypy_path` config option. This option is only useful in - conjunction with :option:`--namespace-packages`. See :ref:`Mapping file + in the absence of `__init__.py`. See :ref:`Mapping file paths to modules ` for details. .. option:: --ignore-missing-imports @@ -236,6 +218,18 @@ imports. setting the :option:`--fast-module-lookup` option. +.. option:: --no-namespace-packages + + This flag disables import discovery of namespace packages (see :pep:`420`). + In particular, this prevents discovery of packages that don't have an + ``__init__.py`` (or ``__init__.pyi``) file. + + This flag affects how mypy finds modules and packages explicitly passed on + the command line. It also affects how mypy determines fully qualified module + names for files passed on the command line. See :ref:`Mapping file paths to + modules ` for details. + + .. _platform-configuration: Platform configuration @@ -390,29 +384,23 @@ None and Optional handling The following flags adjust how mypy handles values of type ``None``. For more details, see :ref:`no_strict_optional`. -.. _no-implicit-optional: +.. _implicit-optional: -.. option:: --no-implicit-optional +.. option:: --implicit-optional - This flag causes mypy to stop treating arguments with a ``None`` + This flag causes mypy to treat arguments with a ``None`` default value as having an implicit :py:data:`~typing.Optional` type. - For example, by default mypy will assume that the ``x`` parameter - is of type ``Optional[int]`` in the code snippet below since - the default parameter is ``None``: + For example, if this flag is set, mypy would assume that the ``x`` + parameter is actually of type ``Optional[int]`` in the code snippet below + since the default parameter is ``None``: .. code-block:: python def foo(x: int = None) -> None: print(x) - If this flag is set, the above snippet will no longer type check: - we must now explicitly indicate that the type is ``Optional[int]``: - - .. code-block:: python - - def foo(x: Optional[int] = None) -> None: - print(x) + **Note:** This was disabled by default starting in mypy 0.980. .. option:: --no-strict-optional @@ -703,9 +691,9 @@ in error messages. ``file:line:column:end_line:end_column``. This option implies ``--show-column-numbers``. -.. option:: --show-error-codes +.. option:: --hide-error-codes - This flag will add an error code ``[]`` to error messages. The error + This flag will hide the error code ``[]`` from error messages. By default, the error code is shown after each error message:: prog.py:1: error: "str" has no attribute "trim" [attr-defined] @@ -830,7 +818,8 @@ in developing or debugging mypy internals. submitting them upstream, but also allows you to use a forked version of typeshed. - Note that this doesn't affect third-party library stubs. + Note that this doesn't affect third-party library stubs. To test third-party stubs, + for example try ``MYPYPATH=stubs/six mypy ...``. .. _warn-incomplete-stub: diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index d2302469518d..42962581702f 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -187,7 +187,13 @@ Ignoring a whole file --------------------- A ``# type: ignore`` comment at the top of a module (before any statements, -including imports or docstrings) has the effect of ignoring the *entire* module. +including imports or docstrings) has the effect of ignoring the entire contents of the module. + +To only ignore errors, use a top-level ``# mypy: ignore-errors`` comment instead. +To only ignore errors with a specific error code, use a top-level +``# mypy: disable-error-code=...`` comment. +To replace the contents of the module with ``Any``, use a per-module ``follow_imports = skip``. +See :ref:`Following imports ` for details. .. code-block:: python diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 663a0d2229a6..abaec31c6888 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -191,6 +191,28 @@ section of the command line docs. This option may only be set in the global section (``[mypy]``). +.. confval:: modules + + :type: comma-separated list of strings + + A comma-separated list of packages which should be checked by mypy if none are given on the command + line. Mypy *will not* recursively type check any submodules of the provided + module. + + This option may only be set in the global section (``[mypy]``). + + +.. confval:: packages + + :type: comma-separated list of strings + + A comma-separated list of packages which should be checked by mypy if none are given on the command + line. Mypy *will* recursively type check any submodules of the provided + package. This flag is identical to :confval:`modules` apart from this + behavior. + + This option may only be set in the global section (``[mypy]``). + .. confval:: exclude :type: regular expression @@ -254,10 +276,11 @@ section of the command line docs. .. confval:: namespace_packages :type: boolean - :default: False + :default: True Enables :pep:`420` style namespace packages. See the - corresponding flag :option:`--namespace-packages ` for more information. + corresponding flag :option:`--no-namespace-packages ` + for more information. This option may only be set in the global section (``[mypy]``). @@ -269,7 +292,7 @@ section of the command line docs. This flag tells mypy that top-level packages will be based in either the current directory, or a member of the ``MYPYPATH`` environment variable or :confval:`mypy_path` config option. This option is only useful in - conjunction with :confval:`namespace_packages`. See :ref:`Mapping file + the absence of `__init__.py`. See :ref:`Mapping file paths to modules ` for details. This option may only be set in the global section (``[mypy]``). @@ -503,13 +526,15 @@ None and Optional handling For more information, see the :ref:`None and Optional handling ` section of the command line docs. -.. confval:: no_implicit_optional +.. confval:: implicit_optional :type: boolean :default: False - Changes the treatment of arguments with a default value of ``None`` by not implicitly - making their type :py:data:`~typing.Optional`. + Causes mypy to treat arguments with a ``None`` + default value as having an implicit :py:data:`~typing.Optional` type. + + **Note:** This was True by default in mypy versions 0.980 and earlier. .. confval:: strict_optional @@ -574,14 +599,6 @@ Suppressing errors Note: these configuration options are available in the config file only. There is no analog available via the command line options. -.. confval:: show_none_errors - - :type: boolean - :default: True - - Shows errors related to strict ``None`` checking, if the global :confval:`strict_optional` - flag is enabled. - .. confval:: ignore_errors :type: boolean @@ -722,12 +739,12 @@ These options may only be set in the global section (``[mypy]``). Shows column numbers in error messages. -.. confval:: show_error_codes +.. confval:: hide_error_codes :type: boolean :default: False - Shows error codes in error messages. See :ref:`error-codes` for more information. + Hides error codes in error messages. See :ref:`error-codes` for more information. .. confval:: pretty @@ -858,9 +875,16 @@ These options may only be set in the global section (``[mypy]``). :type: string - Specifies an alternative directory to look for stubs instead of the - default ``typeshed`` directory. User home directory and environment - variables will be expanded. + This specifies the directory where mypy looks for standard library typeshed + stubs, instead of the typeshed that ships with mypy. This is + primarily intended to make it easier to test typeshed changes before + submitting them upstream, but also allows you to use a forked version of + typeshed. + + User home directory and environment variables will be expanded. + + Note that this doesn't affect third-party library stubs. To test third-party stubs, + for example try ``MYPYPATH=stubs/six mypy ...``. .. confval:: warn_incomplete_stub diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 5c1f0bedb980..264badc03107 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -564,6 +564,54 @@ Example: # Error: Cannot instantiate abstract class "Thing" with abstract attribute "save" [abstract] t = Thing() +Safe handling of abstract type object types [type-abstract] +----------------------------------------------------------- + +Mypy always allows instantiating (calling) type objects typed as ``Type[t]``, +even if it is not known that ``t`` is non-abstract, since it is a common +pattern to create functions that act as object factories (custom constructors). +Therefore, to prevent issues described in the above section, when an abstract +type object is passed where ``Type[t]`` is expected, mypy will give an error. +Example: + +.. code-block:: python + + from abc import ABCMeta, abstractmethod + from typing import List, Type, TypeVar + + class Config(metaclass=ABCMeta): + @abstractmethod + def get_value(self, attr: str) -> str: ... + + T = TypeVar("T") + def make_many(typ: Type[T], n: int) -> List[T]: + return [typ() for _ in range(n)] # This will raise if typ is abstract + + # Error: Only concrete class can be given where "Type[Config]" is expected [type-abstract] + make_many(Config, 5) + +Check that call to an abstract method via super is valid [safe-super] +--------------------------------------------------------------------- + +Abstract methods often don't have any default implementation, i.e. their +bodies are just empty. Calling such methods in subclasses via ``super()`` +will cause runtime errors, so mypy prevents you from doing so: + +.. code-block:: python + + from abc import abstractmethod + class Base: + @abstractmethod + def foo(self) -> int: ... + class Sub(Base): + def foo(self) -> int: + return super().foo() + 1 # error: Call to abstract method "foo" of "Base" with + # trivial body via super() is unsafe [safe-super] + Sub().foo() # This will crash at runtime. + +Mypy considers the following as trivial bodies: a ``pass`` statement, a literal +ellipsis ``...``, a docstring, and a ``raise NotImplementedError`` statement. + Check the target of NewType [valid-newtype] ------------------------------------------- diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index 3938669edafc..cac19e705361 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -258,6 +258,19 @@ except that attempting to invoke an undefined method (e.g. ``__len__``) results while attempting to evaluate an object in boolean context without a concrete implementation results in a truthy value. +Check that function isn't used in boolean context [truthy-function] +------------------------------------------------------------------- + +Functions will always evaluate to true in boolean contexts. + +.. code-block:: python + + def f(): + ... + + if f: # Error: Function "Callable[[], Any]" could always be true in boolean context [truthy-function] + pass + .. _ignore-without-code: Check that ``# type: ignore`` include an error code [ignore-without-code] diff --git a/docs/source/error_codes.rst b/docs/source/error_codes.rst index bed73abc379f..aabedf87f73a 100644 --- a/docs/source/error_codes.rst +++ b/docs/source/error_codes.rst @@ -23,12 +23,12 @@ Error codes may change in future mypy releases. Displaying error codes ---------------------- -Error codes are not displayed by default. Use :option:`--show-error-codes ` -or config ``show_error_codes = True`` to display error codes. Error codes are shown inside square brackets: +Error codes are displayed by default. Use :option:`--hide-error-codes ` +or config ``hide_error_codes = True`` to hide error codes. Error codes are shown inside square brackets: .. code-block:: text - $ mypy --show-error-codes prog.py + $ mypy prog.py prog.py:1: error: "str" has no attribute "trim" [attr-defined] It's also possible to require error codes for ``type: ignore`` comments. @@ -69,3 +69,47 @@ which enables the ``no-untyped-def`` error code. You can use :option:`--enable-error-code ` to enable specific error codes that don't have a dedicated command-line flag or config file setting. + +Per-module enabling/disabling error codes +----------------------------------------- + +You can use :ref:`configuration file ` sections to enable or +disable specific error codes only in some modules. For example, this ``mypy.ini`` +config will enable non-annotated empty containers in tests, while keeping +other parts of code checked in strict mode: + +.. code-block:: ini + + [mypy] + strict = True + + [mypy-tests.*] + allow_untyped_defs = True + allow_untyped_calls = True + disable_error_code = var-annotated, has-type + +Note that per-module enabling/disabling acts as override over the global +options. So that you don't need to repeat the error code lists for each +module if you have them in global config section. For example: + +.. code-block:: ini + + [mypy] + enable_error_code = truthy-bool, ignore-without-code, unused-awaitable + + [mypy-extensions.*] + disable_error_code = unused-awaitable + +The above config will allow unused awaitables in extension modules, but will +still keep the other two error codes enabled. The overall logic is following: + +* Command line and/or config main section set global error codes + +* Individual config sections *adjust* them per glob/module + +* Inline ``# mypy: ...`` comments can further *adjust* them for a specific + module + +So one can e.g. enable some code globally, disable it for all tests in +the corresponding config section, and then re-enable it with an inline +comment in some specific test. diff --git a/docs/source/existing_code.rst b/docs/source/existing_code.rst index 66259e5e94c7..5b1fda40f2d6 100644 --- a/docs/source/existing_code.rst +++ b/docs/source/existing_code.rst @@ -7,38 +7,78 @@ This section explains how to get started using mypy with an existing, significant codebase that has little or no type annotations. If you are a beginner, you can skip this section. -These steps will get you started with mypy on an existing codebase: +Start small +----------- -1. Start small -- get a clean mypy build for some files, with few - annotations +If your codebase is large, pick a subset of your codebase (say, 5,000 to 50,000 +lines) and get mypy to run successfully only on this subset at first, *before +adding annotations*. This should be doable in a day or two. The sooner you get +some form of mypy passing on your codebase, the sooner you benefit. -2. Write a mypy runner script to ensure consistent results +You'll likely need to fix some mypy errors, either by inserting +annotations requested by mypy or by adding ``# type: ignore`` +comments to silence errors you don't want to fix now. -3. Run mypy in Continuous Integration to prevent type errors +We'll mention some tips for getting mypy passing on your codebase in various +sections below. -4. Gradually annotate commonly imported modules +Run mypy consistently and prevent regressions +--------------------------------------------- -5. Write annotations as you modify existing code and write new code +Make sure all developers on your codebase run mypy the same way. +One way to ensure this is adding a small script with your mypy +invocation to your codebase, or adding your mypy invocation to +existing tools you use to run tests, like ``tox``. -6. Use :doc:`monkeytype:index` or `PyAnnotate`_ to automatically annotate legacy code +* Make sure everyone runs mypy with the same options. Checking a mypy + :ref:`configuration file ` into your codebase can help + with this. -We discuss all of these points in some detail below, and a few optional -follow-up steps. +* Make sure everyone type checks the same set of files. See + :ref:`specifying-code-to-be-checked` for details. -Start small ------------ +* Make sure everyone runs mypy with the same version of mypy, for instance + by pinning mypy with the rest of your dev requirements. -If your codebase is large, pick a subset of your codebase (say, 5,000 -to 50,000 lines) and run mypy only on this subset at first, -*without any annotations*. This shouldn't take more than a day or two -to implement, so you start enjoying benefits soon. +In particular, you'll want to make sure to run mypy as part of your +Continuous Integration (CI) system as soon as possible. This will +prevent new type errors from being introduced into your codebase. -You'll likely need to fix some mypy errors, either by inserting -annotations requested by mypy or by adding ``# type: ignore`` -comments to silence errors you don't want to fix now. +A simple CI script could look something like this: + +.. code-block:: text + + python3 -m pip install mypy==0.971 + # Run your standardised mypy invocation, e.g. + mypy my_project + # This could also look like `scripts/run_mypy.sh`, `tox -e mypy`, `make mypy`, etc + +Ignoring errors from certain modules +------------------------------------ -In particular, mypy often generates errors about modules that it can't -find or that don't have stub files: +By default mypy will follow imports in your code and try to check everything. +This means even if you only pass in a few files to mypy, it may still process a +large number of imported files. This could potentially result in lots of errors +you don't want to deal with at the moment. + +One way to deal with this is to ignore errors in modules you aren't yet ready to +type check. The :confval:`ignore_errors` option is useful for this, for instance, +if you aren't yet ready to deal with errors from ``package_to_fix_later``: + +.. code-block:: text + + [mypy-package_to_fix_later.*] + ignore_errors = True + +You could even invert this, by setting ``ignore_errors = True`` in your global +config section and only enabling error reporting with ``ignore_errors = False`` +for the set of modules you are ready to type check. + +Fixing errors related to imports +-------------------------------- + +A common class of error you will encounter is errors from mypy about modules +that it can't find, that don't have types, or don't have stub files: .. code-block:: text @@ -46,7 +86,15 @@ find or that don't have stub files: core/model.py:9: error: Cannot find implementation or library stub for module named 'acme' ... -This is normal, and you can easily ignore these errors. For example, +Sometimes these can be fixed by installing the relevant packages or +stub libraries in the environment you're running ``mypy`` in. + +See :ref:`ignore-missing-imports` for a complete reference on these errors +and the ways in which you can fix them. + +You'll likely find that you want to suppress all errors from importing +a given module that doesn't have types. If you only import that module +in one or two places, you can use ``# type: ignore`` comments. For example, here we ignore an error about a third-party module ``frobnicate`` that doesn't have stubs using ``# type: ignore``: @@ -56,9 +104,9 @@ doesn't have stubs using ``# type: ignore``: ... frobnicate.initialize() # OK (but not checked) -You can also use a mypy configuration file, which is convenient if -there are a large number of errors to ignore. For example, to disable -errors about importing ``frobnicate`` and ``acme`` everywhere in your +But if you import the module in many places, this becomes unwieldy. In this +case, we recommend using a :ref:`configuration file `. For example, +to disable errors about importing ``frobnicate`` and ``acme`` everywhere in your codebase, use a config like this: .. code-block:: text @@ -69,69 +117,33 @@ codebase, use a config like this: [mypy-acme.*] ignore_missing_imports = True -You can add multiple sections for different modules that should be -ignored. - -If your config file is named ``mypy.ini``, this is how you run mypy: - -.. code-block:: text - - mypy --config-file mypy.ini mycode/ - If you get a large number of errors, you may want to ignore all errors -about missing imports. This can easily cause problems later on and -hide real errors, and it's only recommended as a last resort. -For more details, look :ref:`here `. - -Mypy follows imports by default. This can result in a few files passed -on the command line causing mypy to process a large number of imported -files, resulting in lots of errors you don't want to deal with at the -moment. There is a config file option to disable this behavior, but -since this can hide errors, it's not recommended for most users. - -Mypy runner script ------------------- - -Introduce a mypy runner script that runs mypy, so that every developer -will use mypy consistently. Here are some things you may want to do in -the script: - -* Ensure that the correct version of mypy is installed. - -* Specify mypy config file or command-line options. - -* Provide set of files to type check. You may want to implement - inclusion and exclusion filters for full control of the file - list. - -Continuous Integration ----------------------- - -Once you have a clean mypy run and a runner script for a part -of your codebase, set up your Continuous Integration (CI) system to -run mypy to ensure that developers won't introduce bad annotations. -A simple CI script could look something like this: +about missing imports, for instance by setting :confval:`ignore_missing_imports` +to true globally. This can hide errors later on, so we recommend avoiding this +if possible. -.. code-block:: text +Finally, mypy allows fine-grained control over specific import following +behaviour. It's very easy to silently shoot yourself in the foot when playing +around with these, so it's mostly recommended as a last resort. For more +details, look :ref:`here `. - python3 -m pip install mypy==0.790 # Pinned version avoids surprises - scripts/mypy # Run the mypy runner script you set up - -Annotate widely imported modules --------------------------------- +Prioritise annotating widely imported modules +--------------------------------------------- Most projects have some widely imported modules, such as utilities or model classes. It's a good idea to annotate these pretty early on, since this allows code using these modules to be type checked more -effectively. Since mypy supports gradual typing, it's okay to leave -some of these modules unannotated. The more you annotate, the more -useful mypy will be, but even a little annotation coverage is useful. +effectively. + +Mypy is designed to support gradual typing, i.e. letting you add annotations at +your own pace, so it's okay to leave some of these modules unannotated. The more +you annotate, the more useful mypy will be, but even a little annotation +coverage is useful. Write annotations as you go --------------------------- -Now you are ready to include type annotations in your development -workflows. Consider adding something like these in your code style +Consider adding something like these in your code style conventions: 1. Developers should add annotations for any new code. @@ -143,9 +155,9 @@ codebase without much effort. Automate annotation of legacy code ---------------------------------- -There are tools for automatically adding draft annotations -based on type profiles collected at runtime. Tools include -:doc:`monkeytype:index` (Python 3) and `PyAnnotate`_. +There are tools for automatically adding draft annotations based on simple +static analysis or on type profiles collected at runtime. Tools include +:doc:`monkeytype:index`, `autotyping`_ and `PyAnnotate`_. A simple approach is to collect types from test runs. This may work well if your test coverage is good (and if your tests aren't very @@ -156,14 +168,7 @@ fraction of production network requests. This clearly requires more care, as type collection could impact the reliability or the performance of your service. -Speed up mypy runs ------------------- - -You can use :ref:`mypy daemon ` to get much faster -incremental mypy runs. The larger your project is, the more useful -this will be. If your project has at least 100,000 lines of code or -so, you may also want to set up :ref:`remote caching ` -for further speedups. +.. _getting-to-strict: Introduce stricter options -------------------------- @@ -172,7 +177,69 @@ Mypy is very configurable. Once you get started with static typing, you may want to explore the various strictness options mypy provides to catch more bugs. For example, you can ask mypy to require annotations for all functions in certain modules to avoid accidentally introducing code that won't be type checked using -:confval:`disallow_untyped_defs`, or type check code without annotations as well -with :confval:`check_untyped_defs`. Refer to :ref:`config-file` for the details. +:confval:`disallow_untyped_defs`. Refer to :ref:`config-file` for the details. + +An excellent goal to aim for is to have your codebase pass when run against ``mypy --strict``. +This basically ensures that you will never have a type related error without an explicit +circumvention somewhere (such as a ``# type: ignore`` comment). + +The following config is equivalent to ``--strict`` (as of mypy 0.990): + +.. code-block:: text + + # Start off with these + warn_unused_configs = True + warn_redundant_casts = True + warn_unused_ignores = True + no_implicit_optional = True + + # Getting these passing should be easy + strict_equality = True + strict_concatenate = True + + # Strongly recommend enabling this one as soon as you can + check_untyped_defs = True + + # These shouldn't be too much additional work, but may be tricky to + # get passing if you use a lot of untyped libraries + disallow_subclassing_any = True + disallow_untyped_decorators = True + disallow_any_generics = True + + # These next few are various gradations of forcing use of type annotations + disallow_untyped_calls = True + disallow_incomplete_defs = True + disallow_untyped_defs = True + + # This one isn't too hard to get passing, but return on investment is lower + no_implicit_reexport = True + + # This one can be tricky to get passing if you use a lot of untyped libraries + warn_return_any = True + +Note that you can also start with ``--strict`` and subtract, for instance: + +.. code-block:: text + + strict = True + warn_return_any = False + +Remember that many of these options can be enabled on a per-module basis. For instance, +you may want to enable ``disallow_untyped_defs`` for modules which you've completed +annotations for, in order to prevent new code from being added without annotations. + +And if you want, it doesn't stop at ``--strict``. Mypy has additional checks +that are not part of ``--strict`` that can be useful. See the complete +:ref:`command-line` reference and :ref:`error-codes-optional`. + +Speed up mypy runs +------------------ + +You can use :ref:`mypy daemon ` to get much faster +incremental mypy runs. The larger your project is, the more useful +this will be. If your project has at least 100,000 lines of code or +so, you may also want to set up :ref:`remote caching ` +for further speedups. .. _PyAnnotate: https://github.com/dropbox/pyannotate +.. _autotyping: https://github.com/JelleZijlstra/autotyping diff --git a/docs/source/extending_mypy.rst b/docs/source/extending_mypy.rst index 00c328be7728..daf863616334 100644 --- a/docs/source/extending_mypy.rst +++ b/docs/source/extending_mypy.rst @@ -155,23 +155,9 @@ When analyzing this code, mypy will call ``get_type_analyze_hook("lib.Vector")`` so the plugin can return some valid type for each variable. **get_function_hook()** is used to adjust the return type of a function call. -This is a good choice if the return type of some function depends on *values* -of some arguments that can't be expressed using literal types (for example -a function may return an ``int`` for positive arguments and a ``float`` for -negative arguments). This hook will be also called for instantiation of classes. -For example: - -.. code-block:: python - - from contextlib import contextmanager - from typing import TypeVar, Callable - - T = TypeVar('T') - - @contextmanager # built-in plugin can infer a precise type here - def stopwatch(timer: Callable[[], T]) -> Iterator[T]: - ... - yield timer() +This hook will be also called for instantiation of classes. +This is a good choice if the return type is too complex +to be expressed by regular python typing. **get_function_signature_hook** is used to adjust the signature of a function. @@ -251,31 +237,3 @@ mypy's cache for that module so that it can be rechecked. This hook should be used to report to mypy any relevant configuration data, so that mypy knows to recheck the module if the configuration changes. The hooks should return data encodable as JSON. - -Notes about the semantic analyzer -********************************* - -Mypy 0.710 introduced a new semantic analyzer, and the old semantic -analyzer was removed in mypy 0.730. Support for the new semantic analyzer -required some changes to existing plugins. Here is a short summary of the -most important changes: - -* The order of processing AST nodes is different. Code outside - functions is processed first, and functions and methods are - processed afterwards. - -* Each AST node can be processed multiple times to resolve forward - references. The same plugin hook may be called multiple times, so - they need to be idempotent. - -* The ``anal_type()`` API method returns ``None`` if some part of - the type is not available yet due to forward references, for example. - -* When looking up symbols, you may encounter *placeholder nodes* that - are used for names that haven't been fully processed yet. You'll - generally want to request another semantic analysis iteration by - *deferring* in that case. - -See the docstring at the top of -`mypy/plugin.py `_ -for more details. diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index f55a54a0dd30..db7c18d5e242 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -6,15 +6,17 @@ Getting started This chapter introduces some core concepts of mypy, including function annotations, the :py:mod:`typing` module, stub files, and more. -Be sure to read this chapter carefully, as the rest of the documentation +If you're looking for a quick intro, see the +:ref:`mypy cheatsheet `. + +If you're unfamiliar with the concepts of static and dynamic type checking, +be sure to read this chapter carefully, as the rest of the documentation may not make much sense otherwise. Installing and running mypy *************************** -Mypy requires Python 3.6 or later to run. Once you've -`installed Python 3 `_, -install mypy using pip: +Mypy requires Python 3.7 or later to run. You can install mypy using pip: .. code-block:: shell @@ -31,16 +33,19 @@ out any errors it finds. Mypy will type check your code *statically*: this means that it will check for errors without ever running your code, just like a linter. -This means that you are always free to ignore the errors mypy reports and -treat them as just warnings, if you so wish: mypy runs independently from -Python itself. +This also means that you are always free to ignore the errors mypy reports, +if you so wish. You can always use the Python interpreter to run your code, +even if mypy reports errors. However, if you try directly running mypy on your existing Python code, it -will most likely report little to no errors: you must add *type annotations* -to your code to take full advantage of mypy. See the section below for details. +will most likely report little to no errors. This is a feature! It makes it +easy to adopt mypy incrementally. + +In order to get useful diagnostics from mypy, you must add *type annotations* +to your code. See the section below for details. -Function signatures and dynamic vs static typing -************************************************ +Dynamic vs static typing +************************ A function without type annotations is considered to be *dynamically typed* by mypy: @@ -52,22 +57,32 @@ A function without type annotations is considered to be *dynamically typed* by m By default, mypy will **not** type check dynamically typed functions. This means that with a few exceptions, mypy will not report any errors with regular unannotated Python. -This is the case even if you misuse the function: for example, mypy would currently -not report any errors if you tried running ``greeting(3)`` or ``greeting(b"Alice")`` -even though those function calls would result in errors at runtime. +This is the case even if you misuse the function! + +.. code-block:: python + + def greeting(name): + return 'Hello ' + name + + # These calls will fail when the program run, but mypy does not report an error + # because "greeting" does not have type annotations. + greeting(123) + greeting(b"Alice") -You can teach mypy to detect these kinds of bugs by adding *type annotations* (also -known as *type hints*). For example, you can teach mypy that ``greeting`` both accepts +We can get mypy to detect these kinds of bugs by adding *type annotations* (also +known as *type hints*). For example, you can tell mypy that ``greeting`` both accepts and returns a string like so: .. code-block:: python + # The "name: str" annotation says that the "name" argument should be a string + # The "-> str" annotation says that "greeting" will return a string def greeting(name: str) -> str: return 'Hello ' + name -This function is now *statically typed*: mypy can use the provided type hints to detect -incorrect usages of the ``greeting`` function. For example, it will reject the following -calls since the arguments have invalid types: +This function is now *statically typed*: mypy will use the provided type hints +to detect incorrect use of the ``greeting`` function and incorrect use of +variables within the ``greeting`` function. For example: .. code-block:: python @@ -76,9 +91,10 @@ calls since the arguments have invalid types: greeting(3) # Argument 1 to "greeting" has incompatible type "int"; expected "str" greeting(b'Alice') # Argument 1 to "greeting" has incompatible type "bytes"; expected "str" + greeting("World!") # No error -Note that this is all still valid Python 3 code! The function annotation syntax -shown above was added to Python :pep:`as a part of Python 3.0 <3107>`. + def bad_greeting(name: str) -> str: + return 'Hello ' * name # Unsupported operand types for * ("str" and "str") Being able to pick whether you want a function to be dynamically or statically typed can be very helpful. For example, if you are migrating an existing @@ -89,62 +105,32 @@ the code using dynamic typing and only add type hints later once the code is mor Once you are finished migrating or prototyping your code, you can make mypy warn you if you add a dynamic function by mistake by using the :option:`--disallow-untyped-defs ` -flag. See :ref:`command-line` for more information on configuring mypy. - -.. note:: - - The earlier stages of analysis performed by mypy may report errors - even for dynamically typed functions. However, you should not rely - on this, as this may change in the future. - -More function signatures -************************ - -Here are a few more examples of adding type hints to function signatures. - -If a function does not explicitly return a value, give it a return -type of ``None``. Using a ``None`` result in a statically typed -context results in a type check error: +flag. You can also get mypy to provide some limited checking of dynamically typed +functions by using the :option:`--check-untyped-defs ` flag. +See :ref:`command-line` for more information on configuring mypy. -.. code-block:: python - - def p() -> None: - print('hello') - - a = p() # Error: "p" does not return a value - -Make sure to remember to include ``None``: if you don't, the function -will be dynamically typed. For example: - -.. code-block:: python - - def f(): - 1 + 'x' # No static type error (dynamically typed) - - def g() -> None: - 1 + 'x' # Type check error (statically typed) +Strict mode and configuration +***************************** -Arguments with default values can be annotated like so: +Mypy has a *strict mode* that enables a number of additional checks, +like :option:`--disallow-untyped-defs `. -.. code-block:: python - - def greeting(name: str, excited: bool = False) -> str: - message = f'Hello, {name}' - if excited: - message += '!!!' - return message +If you run mypy with the :option:`--strict ` flag, you +will basically never get a type related error at runtime without a corresponding +mypy error, unless you explicitly circumvent mypy somehow. -``*args`` and ``**kwargs`` arguments can be annotated like so: +However, this flag will probably be too aggressive if you are trying +to add static types to a large, existing codebase. See :ref:`existing-code` +for suggestions on how to handle that case. -.. code-block:: python +Mypy is very configurable, so you can start with using ``--strict`` +and toggle off individual checks. For instance, if you use many third +party libraries that do not have types, +:option:`--ignore-missing-imports ` +may be useful. See :ref:`getting-to-strict` for how to build up to ``--strict``. - def stars(*args: int, **kwargs: float) -> None: - # 'args' has type 'tuple[int, ...]' (a tuple of ints) - # 'kwargs' has type 'dict[str, float]' (a dict of strs to floats) - for arg in args: - print(arg) - for key, value in kwargs.items(): - print(key, value) +See :ref:`command-line` and :ref:`config-file` for a complete reference on +configuration options. Additional types, and the typing module *************************************** @@ -436,7 +422,7 @@ often suggest the name of the stub distribution: .. code-block:: text - prog.py:1: error: Library stubs not installed for "yaml" (or incompatible with Python 3.8) + prog.py:1: error: Library stubs not installed for "yaml" prog.py:1: note: Hint: "python3 -m pip install types-PyYAML" ... @@ -444,28 +430,6 @@ You can also :ref:`create stubs ` easily. We discuss strategies for handling errors about missing stubs in :ref:`ignore-missing-imports`. -Configuring mypy -**************** - -Mypy supports many command line options that you can use to tweak how -mypy behaves: see :ref:`command-line` for more details. - -For example, suppose you want to make sure *all* functions within your -codebase are using static typing and make mypy report an error if you -add a dynamically-typed function by mistake. You can make mypy do this -by running mypy with the :option:`--disallow-untyped-defs ` flag. - -Another potentially useful flag is :option:`--strict `, which enables many -(though not all) of the available strictness options -- including -:option:`--disallow-untyped-defs `. - -This flag is mostly useful if you're starting a new project from scratch -and want to maintain a high degree of type safety from day one. However, -this flag will probably be too aggressive if you either plan on using -many untyped third party libraries or are trying to add static types to -a large, existing codebase. See :ref:`existing-code` for more suggestions -on how to handle the latter case. - Next steps ********** diff --git a/docs/source/index.rst b/docs/source/index.rst index 1cd16ff60af9..1f77e951843d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,28 +6,36 @@ Welcome to mypy documentation! ============================== -Mypy is a static type checker for Python 3. If you sprinkle -your code with type annotations, mypy can type check your code and find common -bugs. As mypy is a static analyzer, or a lint-like tool, the type -annotations are just hints for mypy and don't interfere when running your program. -You run your program with a standard Python interpreter, and the annotations -are treated effectively as comments. - -Using the Python 3 annotation syntax (using :pep:`484` and :pep:`526` notation), -you will be able to -efficiently annotate your code and use mypy to check the code for common errors. -Mypy has a powerful and easy-to-use type system with modern features such as -type inference, generics, callable types, tuple types, union types, and -structural subtyping. - -As a developer, you decide how to use mypy in your workflow. You can always -escape to dynamic typing as mypy's approach to static typing doesn't restrict -what you can do in your programs. Using mypy will make your programs easier to -understand, debug, and maintain. +Mypy is a static type checker for Python. + +Type checkers help ensure that you're using variables and functions in your code +correctly. With mypy, add type hints (:pep:`484`) +to your Python programs, and mypy will warn you when you use those types +incorrectly. + +Python is a dynamic language, so usually you'll only see errors in your code +when you attempt to run it. Mypy is a *static* checker, so it finds bugs +in your programs without even running them! + +Here is a small example to whet your appetite: + +.. code-block:: python -This documentation provides a short introduction to mypy. It will help you -get started writing statically typed code. Knowledge of Python and a -statically typed object-oriented language, such as Java, are assumed. + number = input("What is your favourite number?") + print("It is", number + 1) # error: Unsupported operand types for + ("str" and "int") + +Adding type hints for mypy does not interfere with the way your program would +otherwise run. Think of type hints as similar to comments! You can always use +the Python interpreter to run your code, even if mypy reports errors. + +Mypy is designed with gradual typing in mind. This means you can add type +hints to your code base slowly and that you can always fall back to dynamic +typing when static typing is not convenient. + +Mypy has a powerful and easy-to-use type system, supporting features such as +type inference, generics, callable types, tuple types, union types, +structural subtyping and more. Using mypy will make your programs easier to +understand, debug, and maintain. .. note:: diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index d439fe4dc3a6..b9a3b891c99c 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -57,10 +57,10 @@ stubs.) If you have installed typed packages in another Python installation or environment, mypy won't automatically find them. One option is to install another copy of those packages in the environment in which you -use to run mypy. Alternatively, you can use the +installed mypy. Alternatively, you can use the :option:`--python-executable ` flag to point -to the target Python executable, and mypy will find packages installed -for that Python executable. +to the Python executable for another environment, and mypy will find +packages installed for that Python executable. Note that mypy does not support some more advanced import features, such as zip imports and custom import hooks. diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index b9ddaf88ad74..b575a6eac4c5 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -388,12 +388,8 @@ case you should add an explicit ``Optional[...]`` annotation (or type comment). .. note:: ``Optional[...]`` *does not* mean a function argument with a default value. - However, if the default value of an argument is ``None``, you can use - an optional type for the argument, but it's not enforced by default. - You can use the :option:`--no-implicit-optional ` command-line option to stop - treating arguments with a ``None`` default value as having an implicit - ``Optional[...]`` type. It's possible that this will become the default - behavior in the future. + It simply means that ``None`` is a valid value for the argument. This is + a common confusion because ``None`` is a common default value for arguments. .. _alternative_union_syntax: diff --git a/docs/source/metaclasses.rst b/docs/source/metaclasses.rst index a5d16aa722fd..396d7dbb42cc 100644 --- a/docs/source/metaclasses.rst +++ b/docs/source/metaclasses.rst @@ -72,12 +72,15 @@ so it's better not to combine metaclasses and class hierarchies: class A1(metaclass=M1): pass class A2(metaclass=M2): pass - class B1(A1, metaclass=M2): pass # Mypy Error: Inconsistent metaclass structure for "B1" + class B1(A1, metaclass=M2): pass # Mypy Error: metaclass conflict # At runtime the above definition raises an exception # TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases - # Same runtime error as in B1, but mypy does not catch it yet - class B12(A1, A2): pass + class B12(A1, A2): pass # Mypy Error: metaclass conflict + + # This can be solved via a common metaclass subtype: + class CorrectMeta(M1, M2): pass + class B2(A1, A2, metaclass=CorrectMeta): pass # OK, runtime is also OK * Mypy does not understand dynamically-computed metaclasses, such as ``class A(metaclass=f()): ...`` diff --git a/docs/source/more_types.rst b/docs/source/more_types.rst index 722909a038b5..707411e95fef 100644 --- a/docs/source/more_types.rst +++ b/docs/source/more_types.rst @@ -804,10 +804,9 @@ classes are generic, self-type allows giving them precise signatures: .. code-block:: python T = TypeVar('T') + Q = TypeVar('Q', bound='Base[Any]') class Base(Generic[T]): - Q = TypeVar('Q', bound='Base[T]') - def __init__(self, item: T) -> None: self.item = item diff --git a/docs/source/protocols.rst b/docs/source/protocols.rst index 48530310c8cb..603c9fd0dcc8 100644 --- a/docs/source/protocols.rst +++ b/docs/source/protocols.rst @@ -55,11 +55,232 @@ For example, ``IntList`` below is iterable, over ``int`` values: print_numbered(x) # OK print_numbered([4, 5]) # Also OK -The subsections below introduce all built-in protocols defined in +:ref:`predefined_protocols_reference` lists all protocols defined in :py:mod:`typing` and the signatures of the corresponding methods you need to define to implement each protocol (the signatures can be left out, as always, but mypy won't type check unannotated methods). +Simple user-defined protocols +***************************** + +You can define your own protocol class by inheriting the special ``Protocol`` +class: + +.. code-block:: python + + from typing import Iterable + from typing_extensions import Protocol + + class SupportsClose(Protocol): + def close(self) -> None: + ... # Empty method body (explicit '...') + + class Resource: # No SupportsClose base class! + # ... some methods ... + + def close(self) -> None: + self.resource.release() + + def close_all(items: Iterable[SupportsClose]) -> None: + for item in items: + item.close() + + close_all([Resource(), open('some/file')]) # Okay! + +``Resource`` is a subtype of the ``SupportsClose`` protocol since it defines +a compatible ``close`` method. Regular file objects returned by :py:func:`open` are +similarly compatible with the protocol, as they support ``close()``. + +.. note:: + + The ``Protocol`` base class is provided in the ``typing_extensions`` + package for Python 3.4-3.7. Starting with Python 3.8, ``Protocol`` + is included in the ``typing`` module. + +Defining subprotocols and subclassing protocols +*********************************************** + +You can also define subprotocols. Existing protocols can be extended +and merged using multiple inheritance. Example: + +.. code-block:: python + + # ... continuing from the previous example + + class SupportsRead(Protocol): + def read(self, amount: int) -> bytes: ... + + class TaggedReadableResource(SupportsClose, SupportsRead, Protocol): + label: str + + class AdvancedResource(Resource): + def __init__(self, label: str) -> None: + self.label = label + + def read(self, amount: int) -> bytes: + # some implementation + ... + + resource: TaggedReadableResource + resource = AdvancedResource('handle with care') # OK + +Note that inheriting from an existing protocol does not automatically +turn the subclass into a protocol -- it just creates a regular +(non-protocol) class or ABC that implements the given protocol (or +protocols). The ``Protocol`` base class must always be explicitly +present if you are defining a protocol: + +.. code-block:: python + + class NotAProtocol(SupportsClose): # This is NOT a protocol + new_attr: int + + class Concrete: + new_attr: int = 0 + + def close(self) -> None: + ... + + # Error: nominal subtyping used by default + x: NotAProtocol = Concrete() # Error! + +You can also include default implementations of methods in +protocols. If you explicitly subclass these protocols you can inherit +these default implementations. Explicitly including a protocol as a +base class is also a way of documenting that your class implements a +particular protocol, and it forces mypy to verify that your class +implementation is actually compatible with the protocol. In particular, +omitting a value for an attribute or a method body will make it implicitly +abstract: + +.. code-block:: python + + class SomeProto(Protocol): + attr: int # Note, no right hand side + def method(self) -> str: ... # Literal ... here + class ExplicitSubclass(SomeProto): + pass + ExplicitSubclass() # error: Cannot instantiate abstract class 'ExplicitSubclass' + # with abstract attributes 'attr' and 'method' + +Recursive protocols +******************* + +Protocols can be recursive (self-referential) and mutually +recursive. This is useful for declaring abstract recursive collections +such as trees and linked lists: + +.. code-block:: python + + from typing import TypeVar, Optional + from typing_extensions import Protocol + + class TreeLike(Protocol): + value: int + + @property + def left(self) -> Optional['TreeLike']: ... + + @property + def right(self) -> Optional['TreeLike']: ... + + class SimpleTree: + def __init__(self, value: int) -> None: + self.value = value + self.left: Optional['SimpleTree'] = None + self.right: Optional['SimpleTree'] = None + + root: TreeLike = SimpleTree(0) # OK + +Using isinstance() with protocols +********************************* + +You can use a protocol class with :py:func:`isinstance` if you decorate it +with the ``@runtime_checkable`` class decorator. The decorator adds +support for basic runtime structural checks: + +.. code-block:: python + + from typing_extensions import Protocol, runtime_checkable + + @runtime_checkable + class Portable(Protocol): + handles: int + + class Mug: + def __init__(self) -> None: + self.handles = 1 + + def use(handles: int) -> None: ... + + mug = Mug() + if isinstance(mug, Portable): + use(mug.handles) # Works statically and at runtime + +:py:func:`isinstance` also works with the :ref:`predefined protocols ` +in :py:mod:`typing` such as :py:class:`~typing.Iterable`. + +.. note:: + :py:func:`isinstance` with protocols is not completely safe at runtime. + For example, signatures of methods are not checked. The runtime + implementation only checks that all protocol members are defined. + +.. _callback_protocols: + +Callback protocols +****************** + +Protocols can be used to define flexible callback types that are hard +(or even impossible) to express using the :py:data:`Callable[...] ` syntax, such as variadic, +overloaded, and complex generic callbacks. They are defined with a special :py:meth:`__call__ ` +member: + +.. code-block:: python + + from typing import Optional, Iterable + from typing_extensions import Protocol + + class Combiner(Protocol): + def __call__(self, *vals: bytes, maxlen: Optional[int] = None) -> list[bytes]: ... + + def batch_proc(data: Iterable[bytes], cb_results: Combiner) -> bytes: + for item in data: + ... + + def good_cb(*vals: bytes, maxlen: Optional[int] = None) -> list[bytes]: + ... + def bad_cb(*vals: bytes, maxitems: Optional[int]) -> list[bytes]: + ... + + batch_proc([], good_cb) # OK + batch_proc([], bad_cb) # Error! Argument 2 has incompatible type because of + # different name and kind in the callback + +Callback protocols and :py:data:`~typing.Callable` types can be used interchangeably. +Argument names in :py:meth:`__call__ ` methods must be identical, unless +a double underscore prefix is used. For example: + +.. code-block:: python + + from typing import Callable, TypeVar + from typing_extensions import Protocol + + T = TypeVar('T') + + class Copy(Protocol): + def __call__(self, __origin: T) -> T: ... + + copy_a: Callable[[T], T] + copy_b: Copy + + copy_a = copy_b # OK + copy_b = copy_a # Also OK + +.. _predefined_protocols_reference: + +Predefined protocol reference +***************************** + Iteration protocols ................... @@ -283,207 +504,3 @@ AsyncContextManager[T] traceback: Optional[TracebackType]) -> Awaitable[Optional[bool]] See also :py:class:`~typing.AsyncContextManager`. - -Simple user-defined protocols -***************************** - -You can define your own protocol class by inheriting the special ``Protocol`` -class: - -.. code-block:: python - - from typing import Iterable - from typing_extensions import Protocol - - class SupportsClose(Protocol): - def close(self) -> None: - ... # Empty method body (explicit '...') - - class Resource: # No SupportsClose base class! - # ... some methods ... - - def close(self) -> None: - self.resource.release() - - def close_all(items: Iterable[SupportsClose]) -> None: - for item in items: - item.close() - - close_all([Resource(), open('some/file')]) # Okay! - -``Resource`` is a subtype of the ``SupportsClose`` protocol since it defines -a compatible ``close`` method. Regular file objects returned by :py:func:`open` are -similarly compatible with the protocol, as they support ``close()``. - -.. note:: - - The ``Protocol`` base class is provided in the ``typing_extensions`` - package for Python 3.4-3.7. Starting with Python 3.8, ``Protocol`` - is included in the ``typing`` module. - -Defining subprotocols and subclassing protocols -*********************************************** - -You can also define subprotocols. Existing protocols can be extended -and merged using multiple inheritance. Example: - -.. code-block:: python - - # ... continuing from the previous example - - class SupportsRead(Protocol): - def read(self, amount: int) -> bytes: ... - - class TaggedReadableResource(SupportsClose, SupportsRead, Protocol): - label: str - - class AdvancedResource(Resource): - def __init__(self, label: str) -> None: - self.label = label - - def read(self, amount: int) -> bytes: - # some implementation - ... - - resource: TaggedReadableResource - resource = AdvancedResource('handle with care') # OK - -Note that inheriting from an existing protocol does not automatically -turn the subclass into a protocol -- it just creates a regular -(non-protocol) class or ABC that implements the given protocol (or -protocols). The ``Protocol`` base class must always be explicitly -present if you are defining a protocol: - -.. code-block:: python - - class NotAProtocol(SupportsClose): # This is NOT a protocol - new_attr: int - - class Concrete: - new_attr: int = 0 - - def close(self) -> None: - ... - - # Error: nominal subtyping used by default - x: NotAProtocol = Concrete() # Error! - -You can also include default implementations of methods in -protocols. If you explicitly subclass these protocols you can inherit -these default implementations. Explicitly including a protocol as a -base class is also a way of documenting that your class implements a -particular protocol, and it forces mypy to verify that your class -implementation is actually compatible with the protocol. - -Recursive protocols -******************* - -Protocols can be recursive (self-referential) and mutually -recursive. This is useful for declaring abstract recursive collections -such as trees and linked lists: - -.. code-block:: python - - from typing import TypeVar, Optional - from typing_extensions import Protocol - - class TreeLike(Protocol): - value: int - - @property - def left(self) -> Optional['TreeLike']: ... - - @property - def right(self) -> Optional['TreeLike']: ... - - class SimpleTree: - def __init__(self, value: int) -> None: - self.value = value - self.left: Optional['SimpleTree'] = None - self.right: Optional['SimpleTree'] = None - - root: TreeLike = SimpleTree(0) # OK - -Using isinstance() with protocols -********************************* - -You can use a protocol class with :py:func:`isinstance` if you decorate it -with the ``@runtime_checkable`` class decorator. The decorator adds -support for basic runtime structural checks: - -.. code-block:: python - - from typing_extensions import Protocol, runtime_checkable - - @runtime_checkable - class Portable(Protocol): - handles: int - - class Mug: - def __init__(self) -> None: - self.handles = 1 - - def use(handles: int) -> None: ... - - mug = Mug() - if isinstance(mug, Portable): - use(mug.handles) # Works statically and at runtime - -:py:func:`isinstance` also works with the :ref:`predefined protocols ` -in :py:mod:`typing` such as :py:class:`~typing.Iterable`. - -.. note:: - :py:func:`isinstance` with protocols is not completely safe at runtime. - For example, signatures of methods are not checked. The runtime - implementation only checks that all protocol members are defined. - -.. _callback_protocols: - -Callback protocols -****************** - -Protocols can be used to define flexible callback types that are hard -(or even impossible) to express using the :py:data:`Callable[...] ` syntax, such as variadic, -overloaded, and complex generic callbacks. They are defined with a special :py:meth:`__call__ ` -member: - -.. code-block:: python - - from typing import Optional, Iterable - from typing_extensions import Protocol - - class Combiner(Protocol): - def __call__(self, *vals: bytes, maxlen: Optional[int] = None) -> list[bytes]: ... - - def batch_proc(data: Iterable[bytes], cb_results: Combiner) -> bytes: - for item in data: - ... - - def good_cb(*vals: bytes, maxlen: Optional[int] = None) -> list[bytes]: - ... - def bad_cb(*vals: bytes, maxitems: Optional[int]) -> list[bytes]: - ... - - batch_proc([], good_cb) # OK - batch_proc([], bad_cb) # Error! Argument 2 has incompatible type because of - # different name and kind in the callback - -Callback protocols and :py:data:`~typing.Callable` types can be used interchangeably. -Argument names in :py:meth:`__call__ ` methods must be identical, unless -a double underscore prefix is used. For example: - -.. code-block:: python - - from typing import Callable, TypeVar - from typing_extensions import Protocol - - T = TypeVar('T') - - class Copy(Protocol): - def __call__(self, __origin: T) -> T: ... - - copy_a: Callable[[T], T] - copy_b: Copy - - copy_a = copy_b # OK - copy_b = copy_a # Also OK diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index 8e5547ffd374..a7eb3fc5e1e7 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -26,10 +26,6 @@ Specifying code to be checked Mypy lets you specify what files it should type check in several different ways. -Note that if you use namespace packages (in particular, packages without -``__init__.py``), you'll need to specify :option:`--namespace-packages `. - 1. First, you can pass in paths to Python files and directories you want to type check. For example:: @@ -83,6 +79,9 @@ Note that if you use namespace packages (in particular, packages without ...will type check the above string as a mini-program (and in this case, will report that ``list[int]`` is not callable). +You can also use the :confval:`files` option in your :file:`mypy.ini` file to specify which +files to check, in which case you can simply run ``mypy`` with no arguments. + Reading a list of files from a file *********************************** @@ -104,6 +103,82 @@ flags, the recommended approach is to use a :ref:`configuration file ` instead. +.. _mapping-paths-to-modules: + +Mapping file paths to modules +***************************** + +One of the main ways you can tell mypy what to type check +is by providing mypy a list of paths. For example:: + + $ mypy file_1.py foo/file_2.py file_3.pyi some/directory + +This section describes how exactly mypy maps the provided paths +to modules to type check. + +- Mypy will check all paths provided that correspond to files. + +- Mypy will recursively discover and check all files ending in ``.py`` or + ``.pyi`` in directory paths provided, after accounting for + :option:`--exclude `. + +- For each file to be checked, mypy will attempt to associate the file (e.g. + ``project/foo/bar/baz.py``) with a fully qualified module name (e.g. + ``foo.bar.baz``). The directory the package is in (``project``) is then + added to mypy's module search paths. + +How mypy determines fully qualified module names depends on if the options +:option:`--no-namespace-packages ` and +:option:`--explicit-package-bases ` are set. + +1. If :option:`--no-namespace-packages ` is set, + mypy will rely solely upon the presence of ``__init__.py[i]`` files to + determine the fully qualified module name. That is, mypy will crawl up the + directory tree for as long as it continues to find ``__init__.py`` (or + ``__init__.pyi``) files. + + For example, if your directory tree consists of ``pkg/subpkg/mod.py``, mypy + would require ``pkg/__init__.py`` and ``pkg/subpkg/__init__.py`` to exist in + order correctly associate ``mod.py`` with ``pkg.subpkg.mod`` + +2. The default case. If :option:`--namespace-packages ` is on, but :option:`--explicit-package-bases ` is off, mypy will allow for the possibility that + directories without ``__init__.py[i]`` are packages. Specifically, mypy will + look at all parent directories of the file and use the location of the + highest ``__init__.py[i]`` in the directory tree to determine the top-level + package. + + For example, say your directory tree consists solely of ``pkg/__init__.py`` + and ``pkg/a/b/c/d/mod.py``. When determining ``mod.py``'s fully qualified + module name, mypy will look at ``pkg/__init__.py`` and conclude that the + associated module name is ``pkg.a.b.c.d.mod``. + +3. You'll notice that the above case still relies on ``__init__.py``. If + you can't put an ``__init__.py`` in your top-level package, but still wish to + pass paths (as opposed to packages or modules using the ``-p`` or ``-m`` + flags), :option:`--explicit-package-bases ` + provides a solution. + + With :option:`--explicit-package-bases `, mypy + will locate the nearest parent directory that is a member of the ``MYPYPATH`` + environment variable, the :confval:`mypy_path` config or is the current + working directory. Mypy will then use the relative path to determine the + fully qualified module name. + + For example, say your directory tree consists solely of + ``src/namespace_pkg/mod.py``. If you run the following command, mypy + will correctly associate ``mod.py`` with ``namespace_pkg.mod``:: + + $ MYPYPATH=src mypy --namespace-packages --explicit-package-bases . + +If you pass a file not ending in ``.py[i]``, the module name assumed is +``__main__`` (matching the behavior of the Python interpreter), unless +:option:`--scripts-are-modules ` is passed. + +Passing :option:`-v ` will show you the files and associated module +names that mypy will check. + How mypy handles imports ************************ @@ -138,7 +213,7 @@ the import. This can cause errors that look like the following: .. code-block:: text main.py:1: error: Skipping analyzing 'django': module is installed, but missing library stubs or py.typed marker - main.py:2: error: Library stubs not installed for "requests" (or incompatible with Python 3.8) + main.py:2: error: Library stubs not installed for "requests" main.py:3: error: Cannot find implementation or library stub for module named "this_module_does_not_exist" If you get any of these errors on an import, mypy will assume the type of that @@ -243,7 +318,7 @@ the library, you will get a message like this: .. code-block:: text - main.py:1: error: Library stubs not installed for "yaml" (or incompatible with Python 3.8) + main.py:1: error: Library stubs not installed for "yaml" main.py:1: note: Hint: "python3 -m pip install types-PyYAML" main.py:1: note: (or run "mypy --install-types" to install all missing stub packages) @@ -276,6 +351,11 @@ check your code twice -- the first time to find the missing stubs, and the second time to type check your code properly after mypy has installed the stubs. +If you've already installed the relevant third-party libraries in an environment +other than the one mypy is running in, you can use :option:`--python-executable +` flag to point to the Python executable for that +environment, and mypy will find packages installed for that Python executable. + .. _missing-type-hints-for-third-party-library: Cannot find implementation or library stub @@ -314,11 +394,6 @@ this error, try: you must run ``mypy ~/foo-project/src`` (or set the ``MYPYPATH`` to ``~/foo-project/src``. -4. If you are using namespace packages -- packages which do not contain - ``__init__.py`` files within each subfolder -- using the - :option:`--namespace-packages ` command - line flag. - In some rare cases, you may get the "Cannot find implementation or library stub for module" error even when the module is installed in your system. This can happen when the module is both missing type hints and is installed @@ -327,6 +402,66 @@ on your system in an unconventional way. In this case, follow the steps above on how to handle :ref:`missing type hints in third party libraries `. + +.. _finding-imports: + +How imports are found +********************* + +When mypy encounters an ``import`` statement or receives module +names from the command line via the :option:`--module ` or :option:`--package ` +flags, mypy tries to find the module on the file system similar +to the way Python finds it. However, there are some differences. + +First, mypy has its own search path. +This is computed from the following items: + +- The ``MYPYPATH`` environment variable + (a list of directories, colon-separated on UNIX systems, semicolon-separated on Windows). +- The :confval:`mypy_path` config file option. +- The directories containing the sources given on the command line + (see :ref:`Mapping file paths to modules `). +- The installed packages marked as safe for type checking (see + :ref:`PEP 561 support `) +- The relevant directories of the + `typeshed `_ repo. + +.. note:: + + You cannot point to a stub-only package (:pep:`561`) via the ``MYPYPATH``, it must be + installed (see :ref:`PEP 561 support `) + +Second, mypy searches for stub files in addition to regular Python files +and packages. +The rules for searching for a module ``foo`` are as follows: + +- The search looks in each of the directories in the search path + (see above) until a match is found. +- If a package named ``foo`` is found (i.e. a directory + ``foo`` containing an ``__init__.py`` or ``__init__.pyi`` file) + that's a match. +- If a stub file named ``foo.pyi`` is found, that's a match. +- If a Python module named ``foo.py`` is found, that's a match. + +These matches are tried in order, so that if multiple matches are found +in the same directory on the search path +(e.g. a package and a Python file, or a stub file and a Python file) +the first one in the above list wins. + +In particular, if a Python file and a stub file are both present in the +same directory on the search path, only the stub file is used. +(However, if the files are in different directories, the one found +in the earlier directory is used.) + +Setting :confval:`mypy_path`/``MYPYPATH`` is mostly useful in the case +where you want to try running mypy against multiple distinct +sets of files that happen to share some common dependencies. + +For example, if you have multiple projects that happen to be +using the same set of work-in-progress stubs, it could be +convenient to just have your ``MYPYPATH`` point to a single +directory containing the stubs. + .. _follow-imports: Following imports @@ -388,152 +523,3 @@ hard-to-debug errors. Adjusting import following behaviour is often most useful when restricted to specific modules. This can be accomplished by setting a per-module :confval:`follow_imports` config option. - - -.. _mapping-paths-to-modules: - -Mapping file paths to modules -***************************** - -One of the main ways you can tell mypy what to type check -is by providing mypy a list of paths. For example:: - - $ mypy file_1.py foo/file_2.py file_3.pyi some/directory - -This section describes how exactly mypy maps the provided paths -to modules to type check. - -- Mypy will check all paths provided that correspond to files. - -- Mypy will recursively discover and check all files ending in ``.py`` or - ``.pyi`` in directory paths provided, after accounting for - :option:`--exclude `. - -- For each file to be checked, mypy will attempt to associate the file (e.g. - ``project/foo/bar/baz.py``) with a fully qualified module name (e.g. - ``foo.bar.baz``). The directory the package is in (``project``) is then - added to mypy's module search paths. - -How mypy determines fully qualified module names depends on if the options -:option:`--namespace-packages ` and -:option:`--explicit-package-bases ` are set. - -1. If :option:`--namespace-packages ` is off, - mypy will rely solely upon the presence of ``__init__.py[i]`` files to - determine the fully qualified module name. That is, mypy will crawl up the - directory tree for as long as it continues to find ``__init__.py`` (or - ``__init__.pyi``) files. - - For example, if your directory tree consists of ``pkg/subpkg/mod.py``, mypy - would require ``pkg/__init__.py`` and ``pkg/subpkg/__init__.py`` to exist in - order correctly associate ``mod.py`` with ``pkg.subpkg.mod`` - -2. If :option:`--namespace-packages ` is on, but - :option:`--explicit-package-bases ` is off, - mypy will allow for the possibility that directories without - ``__init__.py[i]`` are packages. Specifically, mypy will look at all parent - directories of the file and use the location of the highest - ``__init__.py[i]`` in the directory tree to determine the top-level package. - - For example, say your directory tree consists solely of ``pkg/__init__.py`` - and ``pkg/a/b/c/d/mod.py``. When determining ``mod.py``'s fully qualified - module name, mypy will look at ``pkg/__init__.py`` and conclude that the - associated module name is ``pkg.a.b.c.d.mod``. - -3. You'll notice that the above case still relies on ``__init__.py``. If - you can't put an ``__init__.py`` in your top-level package, but still wish to - pass paths (as opposed to packages or modules using the ``-p`` or ``-m`` - flags), :option:`--explicit-package-bases ` - provides a solution. - - With :option:`--explicit-package-bases `, mypy - will locate the nearest parent directory that is a member of the ``MYPYPATH`` - environment variable, the :confval:`mypy_path` config or is the current - working directory. Mypy will then use the relative path to determine the - fully qualified module name. - - For example, say your directory tree consists solely of - ``src/namespace_pkg/mod.py``. If you run the following command, mypy - will correctly associate ``mod.py`` with ``namespace_pkg.mod``:: - - $ MYPYPATH=src mypy --namespace-packages --explicit-package-bases . - -If you pass a file not ending in ``.py[i]``, the module name assumed is -``__main__`` (matching the behavior of the Python interpreter), unless -:option:`--scripts-are-modules ` is passed. - -Passing :option:`-v ` will show you the files and associated module -names that mypy will check. - - -.. _finding-imports: - -How imports are found -********************* - -When mypy encounters an ``import`` statement or receives module -names from the command line via the :option:`--module ` or :option:`--package ` -flags, mypy tries to find the module on the file system similar -to the way Python finds it. However, there are some differences. - -First, mypy has its own search path. -This is computed from the following items: - -- The ``MYPYPATH`` environment variable - (a colon-separated list of directories). -- The :confval:`mypy_path` config file option. -- The directories containing the sources given on the command line - (see :ref:`Mapping file paths to modules `). -- The installed packages marked as safe for type checking (see - :ref:`PEP 561 support `) -- The relevant directories of the - `typeshed `_ repo. - -.. note:: - - You cannot point to a stub-only package (:pep:`561`) via the ``MYPYPATH``, it must be - installed (see :ref:`PEP 561 support `) - -Second, mypy searches for stub files in addition to regular Python files -and packages. -The rules for searching for a module ``foo`` are as follows: - -- The search looks in each of the directories in the search path - (see above) until a match is found. -- If a package named ``foo`` is found (i.e. a directory - ``foo`` containing an ``__init__.py`` or ``__init__.pyi`` file) - that's a match. -- If a stub file named ``foo.pyi`` is found, that's a match. -- If a Python module named ``foo.py`` is found, that's a match. - -These matches are tried in order, so that if multiple matches are found -in the same directory on the search path -(e.g. a package and a Python file, or a stub file and a Python file) -the first one in the above list wins. - -In particular, if a Python file and a stub file are both present in the -same directory on the search path, only the stub file is used. -(However, if the files are in different directories, the one found -in the earlier directory is used.) - - -Other advice and best practices -******************************* - -There are multiple ways of telling mypy what files to type check, ranging -from passing in command line arguments to using the :confval:`files` or :confval:`mypy_path` -config file options to setting the -``MYPYPATH`` environment variable. - -However, in practice, it is usually sufficient to just use either -command line arguments or the :confval:`files` config file option (the two -are largely interchangeable). - -Setting :confval:`mypy_path`/``MYPYPATH`` is mostly useful in the case -where you want to try running mypy against multiple distinct -sets of files that happen to share some common dependencies. - -For example, if you have multiple projects that happen to be -using the same set of work-in-progress stubs, it could be -convenient to just have your ``MYPYPATH`` point to a single -directory containing the stubs. diff --git a/docs/source/runtime_troubles.rst b/docs/source/runtime_troubles.rst index 1bab66194e47..a62652111de6 100644 --- a/docs/source/runtime_troubles.rst +++ b/docs/source/runtime_troubles.rst @@ -8,8 +8,8 @@ version of Python considers legal code. This section describes these scenarios and explains how to get your code running again. Generally speaking, we have three tools at our disposal: -* For Python 3.7 through 3.9, use of ``from __future__ import annotations`` - (:pep:`563`), made the default in Python 3.11 and later +* Use of ``from __future__ import annotations`` (:pep:`563`) + (this behaviour may eventually be made the default in a future Python version) * Use of string literal types or type comments * Use of ``typing.TYPE_CHECKING`` @@ -18,11 +18,33 @@ problems you may encounter. .. _string-literal-types: -String literal types --------------------- +String literal types and type comments +-------------------------------------- + +Mypy allows you to add type annotations using ``# type:`` type comments. +For example: + +.. code-block:: python + + a = 1 # type: int + + def f(x): # type: (int) -> int + return x + 1 + + # Alternative type comment syntax for functions with many arguments + def send_email( + address, # type: Union[str, List[str]] + sender, # type: str + cc, # type: Optional[List[str]] + subject='', + body=None # type: List[str] + ): + # type: (...) -> bool Type comments can't cause runtime errors because comments are not evaluated by -Python. In a similar way, using string literal types sidesteps the problem of +Python. + +In a similar way, using string literal types sidesteps the problem of annotations that would cause runtime errors. Any type can be entered as a string literal, and you can combine @@ -30,8 +52,8 @@ string-literal types with non-string-literal types freely: .. code-block:: python - def f(a: list['A']) -> None: ... # OK - def g(n: 'int') -> None: ... # OK, though not useful + def f(a: list['A']) -> None: ... # OK, prevents NameError since A is defined later + def g(n: 'int') -> None: ... # Also OK, though not useful class A: pass @@ -47,9 +69,10 @@ Future annotations import (PEP 563) ----------------------------------- Many of the issues described here are caused by Python trying to evaluate -annotations. From Python 3.11 on, Python will no longer attempt to evaluate -function and variable annotations. This behaviour is made available in Python -3.7 and later through the use of ``from __future__ import annotations``. +annotations. Future Python versions (potentially Python 3.12) will by default no +longer attempt to evaluate function and variable annotations. This behaviour is +made available in Python 3.7 and later through the use of +``from __future__ import annotations``. This can be thought of as automatic string literal-ification of all function and variable annotations. Note that function and variable annotations are still @@ -74,7 +97,7 @@ required to be valid Python syntax. For more details, see :pep:`563`. class B: ... class C: ... -.. note:: +.. warning:: Some libraries may have use cases for dynamic evaluation of annotations, for instance, through use of ``typing.get_type_hints`` or ``eval``. If your @@ -273,8 +296,8 @@ the built-in collections or those from :py:mod:`collections.abc`: y: dict[int, str] z: Sequence[str] = x -There is limited support for using this syntax in Python 3.7 and later as well. -If you use ``from __future__ import annotations``, mypy will understand this +There is limited support for using this syntax in Python 3.7 and later as well: +if you use ``from __future__ import annotations``, mypy will understand this syntax in annotations. However, since this will not be supported by the Python interpreter at runtime, make sure you're aware of the caveats mentioned in the notes at :ref:`future annotations import`. @@ -285,8 +308,8 @@ Using X | Y syntax for Unions Starting with Python 3.10 (:pep:`604`), you can spell union types as ``x: int | str``, instead of ``x: typing.Union[int, str]``. -There is limited support for using this syntax in Python 3.7 and later as well. -If you use ``from __future__ import annotations``, mypy will understand this +There is limited support for using this syntax in Python 3.7 and later as well: +if you use ``from __future__ import annotations``, mypy will understand this syntax in annotations, string literal types, type comments and stub files. However, since this will not be supported by the Python interpreter at runtime (if evaluated, ``int | str`` will raise ``TypeError: unsupported operand type(s) diff --git a/docs/source/stubtest.rst b/docs/source/stubtest.rst index ca291f55947e..a8279eb6c239 100644 --- a/docs/source/stubtest.rst +++ b/docs/source/stubtest.rst @@ -41,6 +41,10 @@ stubs and implementation or to check for stub completeness. It's used to test Python's official collection of library stubs, `typeshed `_. +.. warning:: + + stubtest will import and execute Python code from the packages it checks. + Example ******* diff --git a/docs/source/type_inference_and_annotations.rst b/docs/source/type_inference_and_annotations.rst index 47a29a6abf95..5c58d56d85a1 100644 --- a/docs/source/type_inference_and_annotations.rst +++ b/docs/source/type_inference_and_annotations.rst @@ -1,22 +1,35 @@ +.. _type-inference-and-annotations: + Type inference and type annotations =================================== Type inference ************** -Mypy considers the initial assignment as the definition of a variable. -If you do not explicitly -specify the type of the variable, mypy infers the type based on the -static type of the value expression: +For most variables, if you do not explicitly specify its type, mypy will +infer the correct type based on what is initially assigned to the variable. .. code-block:: python - i = 1 # Infer type "int" for i - l = [1, 2] # Infer type "list[int]" for l + # Mypy will infer the type of these variables, despite no annotations + i = 1 + reveal_type(i) # Revealed type is "builtins.int" + l = [1, 2] + reveal_type(l) # Revealed type is "builtins.list[builtins.int]" + + +.. note:: -Type inference is not used in dynamically typed functions (those -without a function type annotation) — every local variable type defaults -to ``Any`` in such functions. ``Any`` is discussed later in more detail. + Note that mypy will not use type inference in dynamically typed functions + (those without a function type annotation) — every local variable type + defaults to ``Any`` in such functions. For more details, see :ref:`dynamic-typing`. + + .. code-block:: python + + def untyped_function(): + i = 1 + reveal_type(i) # Revealed type is "Any" + # 'reveal_type' always outputs 'Any' in unchecked functions .. _explicit-var-types: @@ -35,20 +48,33 @@ variable type annotation: Without the type annotation, the type of ``x`` would be just ``int``. We use an annotation to give it a more general type ``Union[int, str]`` (this type means that the value can be either an ``int`` or a ``str``). -Mypy checks that the type of the initializer is compatible with the -declared type. The following example is not valid, since the initializer is -a floating point number, and this is incompatible with the declared -type: + +The best way to think about this is that the type annotation sets the type of +the variable, not the type of the expression. For instance, mypy will complain +about the following code: .. code-block:: python - x: Union[int, str] = 1.1 # Error! + x: Union[int, str] = 1.1 # error: Incompatible types in assignment + # (expression has type "float", variable has type "Union[int, str]") .. note:: - The best way to think about this is that the type annotation sets the - type of the variable, not the type of the expression. To force the - type of an expression you can use :py:func:`cast(\, \) `. + To explicitly override the type of an expression you can use + :py:func:`cast(\, \) `. + See :ref:`casts` for details. + +Note that you can explicitly declare the type of a variable without +giving it an initial value: + +.. code-block:: python + + # We only unpack two values, so there's no right-hand side value + # for mypy to infer the type of "cs" from: + a, b, *cs = 1, 2 # error: Need type annotation for "cs" + + rs: list[int] # no assignment! + p, q, *rs = 1, 2 # OK Explicit types for collections ****************************** @@ -67,15 +93,9 @@ In these cases you can give the type explicitly using a type annotation: .. code-block:: python - l: list[int] = [] # Create empty list with type list[int] + l: list[int] = [] # Create empty list of int d: dict[str, int] = {} # Create empty dictionary (str -> int) -Similarly, you can also give an explicit type when creating an empty set: - -.. code-block:: python - - s: set[int] = set() - .. note:: Using type arguments (e.g. ``list[int]``) on builtin collections like @@ -88,13 +108,14 @@ Similarly, you can also give an explicit type when creating an empty set: Compatibility of container types ******************************** -The following program generates a mypy error, since ``list[int]`` -is not compatible with ``list[object]``: +A quick note: container types can sometimes be unintuitive. We'll discuss this +more in :ref:`variance`. For example, the following program generates a mypy error, +because mypy treats ``list[int]`` as incompatible with ``list[object]``: .. code-block:: python def f(l: list[object], k: list[int]) -> None: - l = k # Type check error: incompatible types in assignment + l = k # error: Incompatible types in assignment The reason why the above assignment is disallowed is that allowing the assignment could result in non-int values stored in a list of ``int``: @@ -106,33 +127,32 @@ assignment could result in non-int values stored in a list of ``int``: l.append('x') print(k[-1]) # Ouch; a string in list[int] -Other container types like :py:class:`dict` and :py:class:`set` behave similarly. We -will discuss how you can work around this in :ref:`variance`. +Other container types like :py:class:`dict` and :py:class:`set` behave similarly. -You can still run the above program; it prints ``x``. This illustrates -the fact that static types are used during type checking, but they do -not affect the runtime behavior of programs. You can run programs with -type check failures, which is often very handy when performing a large -refactoring. Thus you can always 'work around' the type system, and it +You can still run the above program; it prints ``x``. This illustrates the fact +that static types do not affect the runtime behavior of programs. You can run +programs with type check failures, which is often very handy when performing a +large refactoring. Thus you can always 'work around' the type system, and it doesn't really limit what you can do in your program. Context in type inference ************************* -Type inference is *bidirectional* and takes context into account. For -example, the following is valid: +Type inference is *bidirectional* and takes context into account. + +Mypy will take into account the type of the variable on the left-hand side +of an assignment when inferring the type of the expression on the right-hand +side. For example, the following will type check: .. code-block:: python def f(l: list[object]) -> None: l = [1, 2] # Infer type list[object] for [1, 2], not list[int] -In an assignment, the type context is determined by the assignment -target. In this case this is ``l``, which has the type -``list[object]``. The value expression ``[1, 2]`` is type checked in -this context and given the type ``list[object]``. In the previous -example we introduced a new variable ``l``, and here the type context -was empty. + +The value expression ``[1, 2]`` is type checked with the additional +context that it is being assigned to a variable of type ``list[object]``. +This is used to infer the type of the *expression* as ``list[object]``. Declared argument types are also used for type context. In this program mypy knows that the empty list ``[]`` should have type ``list[int]`` based @@ -165,51 +185,30 @@ Working around the issue is easy by adding a type annotation: a: list[int] = [] # OK foo(a) -Starred expressions -******************* - -In most cases, mypy can infer the type of starred expressions from the -right-hand side of an assignment, but not always: - -.. code-block:: python - - a, *bs = 1, 2, 3 # OK - p, q, *rs = 1, 2 # Error: Type of rs cannot be inferred - -On first line, the type of ``bs`` is inferred to be -``list[int]``. However, on the second line, mypy cannot infer the type -of ``rs``, because there is no right-hand side value for ``rs`` to -infer the type from. In cases like these, the starred expression needs -to be annotated with a starred type: - -.. code-block:: python - - p, q, *rs = 1, 2 # type: int, int, list[int] - -Here, the type of ``rs`` is set to ``list[int]``. - Silencing type errors ********************* You might want to disable type checking on specific lines, or within specific files in your codebase. To do that, you can use a ``# type: ignore`` comment. -For example, say that the web framework that you use now takes an integer -argument to ``run()``, which starts it on localhost on that port. Like so: +For example, say in its latest update, the web framework you use can now take an +integer argument to ``run()``, which starts it on localhost on that port. +Like so: .. code-block:: python # Starting app on http://localhost:8000 app.run(8000) -However, the type stubs that the package uses is not up-to-date, and it still -expects only ``str`` types for ``run()``. This would give you the following error: +However, the devs forgot to update their type annotations for +``run``, so mypy still thinks ``run`` only expects ``str`` types. +This would give you the following error: .. code-block:: text error: Argument 1 to "run" of "A" has incompatible type "int"; expected "str" -If you cannot directly fix the type stubs yourself, you can temporarily +If you cannot directly fix the web framework yourself, you can temporarily disable type checking on that line, by adding a ``# type: ignore``: .. code-block:: python @@ -227,11 +226,10 @@ short explanation of the bug. To do that, use this format: .. code-block:: python # Starting app on http://localhost:8000 - app.run(8000) # type: ignore # `run()` now accepts an `int`, as a port + app.run(8000) # type: ignore # `run()` in v2.0 accepts an `int`, as a port -Mypy displays an error code for each error if you use -:option:`--show-error-codes `: +By default, mypy displays an error code for each error: .. code-block:: text @@ -242,12 +240,12 @@ It is possible to add a specific error-code in your ignore comment (e.g. ``# type: ignore[attr-defined]``) to clarify what's being silenced. You can find more information about error codes :ref:`here `. -Similarly, you can also ignore all mypy checks in a file, by adding a -``# type: ignore`` at the top of the file: +Similarly, you can also ignore all mypy errors in a file, by adding a +``# mypy: ignore-errors`` at the top of the file: .. code-block:: python - # type: ignore + # mypy: ignore-errors # This is a test file, skipping type checking in it. import unittest ... diff --git a/misc/actions_stubs.py b/misc/actions_stubs.py deleted file mode 100644 index 3b13c5d28820..000000000000 --- a/misc/actions_stubs.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import os -import shutil -from typing import Any - -try: - import click -except ImportError: - print("You need the module 'click'") - exit(1) - -base_path = os.getcwd() - -# I don't know how to set callables with different args -def apply_all( - func: Any, - directory: str, - extension: str, - to_extension: str = "", - exclude: tuple[str] = ("",), - recursive: bool = True, - debug: bool = False, -) -> None: - excluded = [x + extension for x in exclude] if exclude else [] - for p, d, files in os.walk(os.path.join(base_path, directory)): - for f in files: - if f in excluded: - continue - inner_path = os.path.join(p, f) - if not inner_path.endswith(extension): - continue - if to_extension: - new_path = f"{inner_path[:-len(extension)]}{to_extension}" - func(inner_path, new_path) - else: - func(inner_path) - if not recursive: - break - - -def confirm(resp: bool = False, **kargs) -> bool: - kargs["rest"] = "to this {f2}/*{e2}".format(**kargs) if kargs.get("f2") else "" - prompt = "{act} all files {rec}matching this expression {f1}/*{e1} {rest}".format(**kargs) - prompt.format(**kargs) - prompt = "{} [{}]|{}: ".format(prompt, "Y" if resp else "N", "n" if resp else "y") - while True: - ans = input(prompt).lower() - if not ans: - return resp - if ans not in ["y", "n"]: - print("Please, enter (y) or (n).") - continue - if ans == "y": - return True - else: - return False - - -actions = ["cp", "mv", "rm"] - - -@click.command(context_settings=dict(help_option_names=["-h", "--help"])) -@click.option( - "--action", "-a", type=click.Choice(actions), required=True, help="What do I have to do :-)" -) -@click.option("--dir", "-d", "directory", default="stubs", help="Directory to start search!") -@click.option( - "--ext", - "-e", - "extension", - default=".py", - help='Extension "from" will be applied the action. Default .py', -) -@click.option( - "--to", - "-t", - "to_extension", - default=".pyi", - help='Extension "to" will be applied the action if can. Default .pyi', -) -@click.option( - "--exclude", - "-x", - multiple=True, - default=("__init__",), - help="For every appear, will ignore this files. (can set multiples times)", -) -@click.option( - "--not-recursive", - "-n", - default=True, - is_flag=True, - help="Set if don't want to walk recursively.", -) -def main( - action: str, - directory: str, - extension: str, - to_extension: str, - exclude: tuple[str], - not_recursive: bool, -) -> None: - """ - This script helps to copy/move/remove files based on their extension. - - The three actions will ask you for confirmation. - - Examples (by default the script search in stubs directory): - - - Change extension of all stubs from .py to .pyi: - - python -a mv - - - Revert the previous action. - - python -a mv -e .pyi -t .py - - - If you want to ignore "awesome.py" files. - - python -a [cp|mv|rm] -x awesome - - - If you want to ignore "awesome.py" and "__init__.py" files. - - python -a [cp|mv|rm] -x awesome -x __init__ - - - If you want to remove all ".todo" files in "todo" directory, but not recursively: - - python -a rm -e .todo -d todo -r - - """ - if action not in actions: - print("Your action have to be one of this: {}".format(", ".join(actions))) - return - - rec = "[Recursively] " if not_recursive else "" - if not extension.startswith("."): - extension = f".{extension}" - if not to_extension.startswith("."): - to_extension = f".{to_extension}" - if directory.endswith("/"): - directory = directory[:-1] - if action == "cp": - if confirm(act="Copy", rec=rec, f1=directory, e1=extension, f2=directory, e2=to_extension): - apply_all(shutil.copy, directory, extension, to_extension, exclude, not_recursive) - elif action == "rm": - if confirm(act="Remove", rec=rec, f1=directory, e1=extension): - apply_all(os.remove, directory, extension, exclude=exclude, recursive=not_recursive) - elif action == "mv": - if confirm(act="Move", rec=rec, f1=directory, e1=extension, f2=directory, e2=to_extension): - apply_all(shutil.move, directory, extension, to_extension, exclude, not_recursive) - - -if __name__ == "__main__": - main() diff --git a/misc/analyze_cache.py b/misc/analyze_cache.py index 25ae9713f6f1..8b805d8da0bc 100644 --- a/misc/analyze_cache.py +++ b/misc/analyze_cache.py @@ -30,7 +30,7 @@ def __init__( self.meta_size = meta_size @property - def total_size(self): + def total_size(self) -> int: return self.data_size + self.meta_size @@ -75,7 +75,7 @@ def pluck(name: str, chunks: Iterable[JsonDict]) -> Iterable[JsonDict]: return (chunk for chunk in chunks if chunk[".class"] == name) -def report_counter(counter: Counter, amount: int | None = None) -> None: +def report_counter(counter: Counter[str], amount: int | None = None) -> None: for name, count in counter.most_common(amount): print(f" {count: <8} {name}") print() @@ -89,7 +89,7 @@ def compress(chunk: JsonDict) -> JsonDict: cache: dict[int, JsonDict] = {} counter = 0 - def helper(chunk: Any) -> Any: + def helper(chunk: JsonDict) -> JsonDict: nonlocal counter if not isinstance(chunk, dict): return chunk @@ -121,7 +121,7 @@ def helper(chunk: Any) -> Any: def decompress(chunk: JsonDict) -> JsonDict: cache: dict[int, JsonDict] = {} - def helper(chunk: Any) -> Any: + def helper(chunk: JsonDict) -> JsonDict: if not isinstance(chunk, dict): return chunk if ".id" in chunk: @@ -167,6 +167,7 @@ def main() -> None: if "build.*.json" in chunk.filename: build = chunk break + assert build is not None original = json.dumps(build.data, sort_keys=True) print(f"Size of build.data.json, in kilobytes: {len(original) / 1024:.3f}") diff --git a/misc/async_matrix.py b/misc/async_matrix.py index ba04fc390069..d4612dd81799 100644 --- a/misc/async_matrix.py +++ b/misc/async_matrix.py @@ -70,7 +70,7 @@ def plain_host_generator(func) -> Generator[str, None, None]: x = 0 f = func() try: - x = yield from f + x = yield from f # noqa: F841 finally: try: f.close() @@ -80,7 +80,7 @@ def plain_host_generator(func) -> Generator[str, None, None]: async def plain_host_coroutine(func) -> None: x = 0 - x = await func() + x = await func() # noqa: F841 @coroutine @@ -89,7 +89,7 @@ def decorated_host_generator(func) -> Generator[str, None, None]: x = 0 f = func() try: - x = yield from f + x = yield from f # noqa: F841 finally: try: f.close() @@ -100,13 +100,13 @@ def decorated_host_generator(func) -> Generator[str, None, None]: @coroutine async def decorated_host_coroutine(func) -> None: x = 0 - x = await func() + x = await func() # noqa: F841 # Main driver. -def main(): +def main() -> None: verbose = "-v" in sys.argv for host in [ plain_host_generator, diff --git a/misc/build-debug-python.sh b/misc/build-debug-python.sh index 2f32a46ce885..f652d6ad9937 100755 --- a/misc/build-debug-python.sh +++ b/misc/build-debug-python.sh @@ -1,7 +1,7 @@ #!/bin/bash -eux # Build a debug build of python, install it, and create a venv for it -# This is mainly intended for use in our travis builds but it can work +# This is mainly intended for use in our github actions builds but it can work # locally. (Though it unfortunately uses brew on OS X to deal with openssl # nonsense.) # Usage: build-debug-python.sh diff --git a/misc/cherry-pick-typeshed.py b/misc/cherry-pick-typeshed.py index 3cf826533a94..af08009c2a8f 100644 --- a/misc/cherry-pick-typeshed.py +++ b/misc/cherry-pick-typeshed.py @@ -37,7 +37,7 @@ def main() -> None: sys.exit(f"error: Invalid commit {commit!r}") if not os.path.exists("mypy") or not os.path.exists("mypyc"): - sys.exit(f"error: This script must be run at the mypy repository root directory") + sys.exit("error: This script must be run at the mypy repository root directory") with tempfile.TemporaryDirectory() as d: diff_file = os.path.join(d, "diff") diff --git a/misc/convert-cache.py b/misc/convert-cache.py index 92a313c6f2a0..e5da9c2650d5 100755 --- a/misc/convert-cache.py +++ b/misc/convert-cache.py @@ -14,7 +14,7 @@ import argparse -from mypy.metastore import FilesystemMetadataStore, SqliteMetadataStore +from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore def main() -> None: @@ -37,7 +37,8 @@ def main() -> None: input_dir = args.input_dir output_dir = args.output_dir or input_dir if args.to_sqlite: - input, output = FilesystemMetadataStore(input_dir), SqliteMetadataStore(output_dir) + input: MetadataStore = FilesystemMetadataStore(input_dir) + output: MetadataStore = SqliteMetadataStore(output_dir) else: input, output = SqliteMetadataStore(input_dir), FilesystemMetadataStore(output_dir) diff --git a/scripts/find_type.py b/misc/find_type.py similarity index 95% rename from scripts/find_type.py rename to misc/find_type.py index 7bded322e6e5..0031c72aea9f 100755 --- a/scripts/find_type.py +++ b/misc/find_type.py @@ -17,7 +17,7 @@ # " Convert to 0-based column offsets # let startcol = startcol - 1 # " Change this line to point to the find_type.py script. -# execute '!python3 /path/to/mypy/scripts/find_type.py % ' . startline . ' ' . startcol . ' ' . endline . ' ' . endcol . ' ' . mypycmd +# execute '!python3 /path/to/mypy/misc/find_type.py % ' . startline . ' ' . startcol . ' ' . endline . ' ' . endcol . ' ' . mypycmd # endfunction # vnoremap t :call RevealType() # @@ -68,7 +68,7 @@ def process_output(output: str, filename: str, start_line: int) -> tuple[str | N return None, True # finding no reveal_type is an error -def main(): +def main() -> None: filename, start_line_str, start_col_str, end_line_str, end_col_str, *mypy_and_args = sys.argv[ 1: ] diff --git a/misc/fix_annotate.py b/misc/fix_annotate.py index 7148b69259be..b661a899924c 100644 --- a/misc/fix_annotate.py +++ b/misc/fix_annotate.py @@ -72,12 +72,12 @@ def transform(self, node, results): # # "Compact" functions (e.g. "def foo(x, y): return max(x, y)") # have a different structure that isn't matched by PATTERN. - - ## print('-'*60) - ## print(node) - ## for i, ch in enumerate(children): - ## print(i, repr(ch.prefix), repr(ch)) - + # + # print('-'*60) + # print(node) + # for i, ch in enumerate(children): + # print(i, repr(ch.prefix), repr(ch)) + # # Check if there's already an annotation. for ch in children: if ch.prefix.lstrip().startswith("# type:"): diff --git a/misc/incremental_checker.py b/misc/incremental_checker.py index 12dc37e2f05e..85239b6462b8 100755 --- a/misc/incremental_checker.py +++ b/misc/incremental_checker.py @@ -44,8 +44,8 @@ import textwrap import time from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter -from typing import Any, Dict, Tuple -from typing_extensions import TypeAlias as _TypeAlias +from typing import Any, Dict +from typing_extensions import Final, TypeAlias as _TypeAlias CACHE_PATH: Final = ".incremental_checker_cache.json" MYPY_REPO_URL: Final = "https://github.com/python/mypy.git" @@ -70,7 +70,7 @@ def execute(command: list[str], fail_on_error: bool = True) -> tuple[str, str, i proc = subprocess.Popen( " ".join(command), stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True ) - stdout_bytes, stderr_bytes = proc.communicate() # type: Tuple[bytes, bytes] + stdout_bytes, stderr_bytes = proc.communicate() stdout, stderr = stdout_bytes.decode("utf-8"), stderr_bytes.decode("utf-8") if fail_on_error and proc.returncode != 0: print("EXECUTED COMMAND:", repr(command)) @@ -197,7 +197,9 @@ def stop_daemon() -> None: def load_cache(incremental_cache_path: str = CACHE_PATH) -> JsonDict: if os.path.exists(incremental_cache_path): with open(incremental_cache_path) as stream: - return json.load(stream) + cache = json.load(stream) + assert isinstance(cache, dict) + return cache else: return {} @@ -405,7 +407,7 @@ def main() -> None: parser.add_argument( "range_start", metavar="COMMIT_ID_OR_NUMBER", - help="the commit id to start from, or the number of " "commits to move back (see above)", + help="the commit id to start from, or the number of commits to move back (see above)", ) parser.add_argument( "-r", @@ -437,7 +439,7 @@ def main() -> None: "--branch", default=None, metavar="NAME", - help="check out and test a custom branch" "uses the default if not specified", + help="check out and test a custom branch uses the default if not specified", ) parser.add_argument("--sample", type=int, help="use a random sample of size SAMPLE") parser.add_argument("--seed", type=str, help="random seed") diff --git a/misc/macs.el b/misc/macs.el index 67d80aa575b0..f4cf6702b989 100644 --- a/misc/macs.el +++ b/misc/macs.el @@ -11,7 +11,7 @@ (thereline (line-number-at-pos there)) (therecol (save-excursion (goto-char there) (current-column)))) (shell-command - (format "cd ~/src/mypy; python3 ./scripts/find_type.py %s %s %s %s %s python3 -m mypy -i mypy" + (format "cd ~/src/mypy; python3 ./misc/find_type.py %s %s %s %s %s python3 -m mypy -i mypy" filename hereline herecol thereline therecol) ) ) diff --git a/misc/perf_checker.py b/misc/perf_checker.py index 52095f9fe052..20c313e61af9 100644 --- a/misc/perf_checker.py +++ b/misc/perf_checker.py @@ -8,7 +8,7 @@ import subprocess import textwrap import time -from typing import Callable, Tuple +from typing import Callable class Command: @@ -32,7 +32,7 @@ def execute(command: list[str]) -> None: proc = subprocess.Popen( " ".join(command), stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True ) - stdout_bytes, stderr_bytes = proc.communicate() # type: Tuple[bytes, bytes] + stdout_bytes, stderr_bytes = proc.communicate() stdout, stderr = stdout_bytes.decode("utf-8"), stderr_bytes.decode("utf-8") if proc.returncode != 0: print("EXECUTED COMMAND:", repr(command)) diff --git a/misc/proper_plugin.py b/misc/proper_plugin.py index 75f6417a3574..a8a8e80ef360 100644 --- a/misc/proper_plugin.py +++ b/misc/proper_plugin.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import Callable, Type as typing_Type +from typing import Callable +from mypy.checker import TypeChecker from mypy.nodes import TypeInfo from mypy.plugin import FunctionContext, Plugin from mypy.subtypes import is_proper_subtype @@ -50,10 +51,8 @@ def isinstance_proper_hook(ctx: FunctionContext) -> Type: right = get_proper_type(ctx.arg_types[1][0]) for arg in ctx.arg_types[0]: if ( - is_improper_type(arg) - or isinstance(get_proper_type(arg), AnyType) - and is_dangerous_target(right) - ): + is_improper_type(arg) or isinstance(get_proper_type(arg), AnyType) + ) and is_dangerous_target(right): if is_special_target(right): return ctx.default_return_type ctx.api.fail( @@ -155,11 +154,13 @@ def proper_types_hook(ctx: FunctionContext) -> Type: def get_proper_type_instance(ctx: FunctionContext) -> Instance: - types = ctx.api.modules["mypy.types"] # type: ignore + checker = ctx.api + assert isinstance(checker, TypeChecker) + types = checker.modules["mypy.types"] proper_type_info = types.names["ProperType"] assert isinstance(proper_type_info.node, TypeInfo) return Instance(proper_type_info.node, []) -def plugin(version: str) -> typing_Type[ProperTypePlugin]: +def plugin(version: str) -> type[ProperTypePlugin]: return ProperTypePlugin diff --git a/misc/sync-typeshed.py b/misc/sync-typeshed.py index 05202b989585..c6856f86744a 100644 --- a/misc/sync-typeshed.py +++ b/misc/sync-typeshed.py @@ -10,16 +10,21 @@ from __future__ import annotations import argparse +import functools import os +import re import shutil import subprocess import sys import tempfile import textwrap +from collections.abc import Mapping + +import requests def check_state() -> None: - if not os.path.isfile("README.md"): + if not os.path.isfile("README.md") and not os.path.isdir("mypy"): sys.exit("error: The current working directory must be the mypy repository root") out = subprocess.check_output(["git", "status", "-s", os.path.join("mypy", "typeshed")]) if out: @@ -37,6 +42,7 @@ def update_typeshed(typeshed_dir: str, commit: str | None) -> str: if commit: subprocess.run(["git", "checkout", commit], check=True, cwd=typeshed_dir) commit = git_head_commit(typeshed_dir) + stdlib_dir = os.path.join("mypy", "typeshed", "stdlib") # Remove existing stubs. shutil.rmtree(stdlib_dir) @@ -60,6 +66,69 @@ def git_head_commit(repo: str) -> str: return commit.strip() +@functools.cache +def get_github_api_headers() -> Mapping[str, str]: + headers = {"Accept": "application/vnd.github.v3+json"} + secret = os.environ.get("GITHUB_TOKEN") + if secret is not None: + headers["Authorization"] = ( + f"token {secret}" if secret.startswith("ghp") else f"Bearer {secret}" + ) + return headers + + +@functools.cache +def get_origin_owner() -> str: + output = subprocess.check_output(["git", "remote", "get-url", "origin"], text=True).strip() + match = re.match( + r"(git@github.com:|https://github.com/)(?P[^/]+)/(?P[^/\s]+)", output + ) + assert match is not None, f"Couldn't identify origin's owner: {output!r}" + assert ( + match.group("repo").removesuffix(".git") == "mypy" + ), f'Unexpected repo: {match.group("repo")!r}' + return match.group("owner") + + +def create_or_update_pull_request(*, title: str, body: str, branch_name: str) -> None: + fork_owner = get_origin_owner() + + with requests.post( + "https://api.github.com/repos/python/mypy/pulls", + json={ + "title": title, + "body": body, + "head": f"{fork_owner}:{branch_name}", + "base": "master", + }, + headers=get_github_api_headers(), + ) as response: + resp_json = response.json() + if response.status_code == 422 and any( + "A pull request already exists" in e.get("message", "") + for e in resp_json.get("errors", []) + ): + # Find the existing PR + with requests.get( + "https://api.github.com/repos/python/mypy/pulls", + params={"state": "open", "head": f"{fork_owner}:{branch_name}", "base": "master"}, + headers=get_github_api_headers(), + ) as response: + response.raise_for_status() + resp_json = response.json() + assert len(resp_json) >= 1 + pr_number = resp_json[0]["number"] + # Update the PR's title and body + with requests.patch( + f"https://api.github.com/repos/python/mypy/pulls/{pr_number}", + json={"title": title, "body": body}, + headers=get_github_api_headers(), + ) as response: + response.raise_for_status() + return + response.raise_for_status() + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( @@ -72,12 +141,21 @@ def main() -> None: default=None, help="Location of typeshed (default to a temporary repository clone)", ) + parser.add_argument( + "--make-pr", + action="store_true", + help="Whether to make a PR with the changes (default to no)", + ) args = parser.parse_args() + check_state() - print("Update contents of mypy/typeshed from typeshed? [yN] ", end="") - answer = input() - if answer.lower() != "y": - sys.exit("Aborting") + + if args.make_pr: + if os.environ.get("GITHUB_TOKEN") is None: + raise ValueError("GITHUB_TOKEN environment variable must be set") + + branch_name = "mypybot/sync-typeshed" + subprocess.run(["git", "checkout", "-B", branch_name, "origin/master"], check=True) if not args.typeshed_dir: # Clone typeshed repo if no directory given. @@ -95,19 +173,34 @@ def main() -> None: # Create a commit message = textwrap.dedent( - """\ + f"""\ Sync typeshed Source commit: https://github.com/python/typeshed/commit/{commit} - """.format( - commit=commit - ) + """ ) subprocess.run(["git", "add", "--all", os.path.join("mypy", "typeshed")], check=True) subprocess.run(["git", "commit", "-m", message], check=True) print("Created typeshed sync commit.") + # Currently just LiteralString reverts + commits_to_cherry_pick = ["780534b13722b7b0422178c049a1cbbf4ea4255b"] + for commit in commits_to_cherry_pick: + subprocess.run(["git", "cherry-pick", commit], check=True) + print(f"Cherry-picked {commit}.") + + if args.make_pr: + subprocess.run(["git", "push", "--force", "origin", branch_name], check=True) + print("Pushed commit.") + + warning = "Note that you will need to close and re-open the PR in order to trigger CI." + + create_or_update_pull_request( + title="Sync typeshed", body=message + "\n" + warning, branch_name=branch_name + ) + print("Created PR.") + if __name__ == "__main__": main() diff --git a/misc/test_case_to_actual.py b/misc/test_case_to_actual.py deleted file mode 100644 index 92d11866ef9d..000000000000 --- a/misc/test_case_to_actual.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -import os -import os.path -import sys -from typing import Iterator - - -class Chunk: - def __init__(self, header_type: str, args: str) -> None: - self.header_type = header_type - self.args = args - self.lines: list[str] = [] - - -def is_header(line: str) -> bool: - return line.startswith("[") and line.endswith("]") - - -def normalize(lines: Iterator[str]) -> Iterator[str]: - return (line.rstrip() for line in lines) - - -def produce_chunks(lines: Iterator[str]) -> Iterator[Chunk]: - current_chunk: Chunk = None - for line in normalize(lines): - if is_header(line): - if current_chunk is not None: - yield current_chunk - parts = line[1:-1].split(" ", 1) - args = parts[1] if len(parts) > 1 else "" - current_chunk = Chunk(parts[0], args) - else: - current_chunk.lines.append(line) - if current_chunk is not None: - yield current_chunk - - -def write_out(filename: str, lines: list[str]) -> None: - os.makedirs(os.path.dirname(filename), exist_ok=True) - with open(filename, "w") as stream: - stream.write("\n".join(lines)) - - -def write_tree(root: str, chunks: Iterator[Chunk]) -> None: - init = next(chunks) - assert init.header_type == "case" - - root = os.path.join(root, init.args) - write_out(os.path.join(root, "main.py"), init.lines) - - for chunk in chunks: - if chunk.header_type == "file" and chunk.args.endswith(".py"): - write_out(os.path.join(root, chunk.args), chunk.lines) - - -def help() -> None: - print("Usage: python misc/test_case_to_actual.py test_file.txt root_path") - - -def main() -> None: - if len(sys.argv) != 3: - help() - return - - test_file_path, root_path = sys.argv[1], sys.argv[2] - with open(test_file_path) as stream: - chunks = produce_chunks(iter(stream)) - write_tree(root_path, chunks) - - -if __name__ == "__main__": - main() diff --git a/misc/touch_checker.py b/misc/touch_checker.py deleted file mode 100644 index 2adcacc3af9a..000000000000 --- a/misc/touch_checker.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import glob -import os -import shutil -import statistics -import subprocess -import sys -import textwrap -import time -from typing import Callable, Tuple - - -def print_offset(text: str, indent_length: int = 4) -> None: - print() - print(textwrap.indent(text, " " * indent_length)) - print() - - -def delete_folder(folder_path: str) -> None: - if os.path.exists(folder_path): - shutil.rmtree(folder_path) - - -def execute(command: list[str]) -> None: - proc = subprocess.Popen( - " ".join(command), stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True - ) - stdout_bytes, stderr_bytes = proc.communicate() # type: Tuple[bytes, bytes] - stdout, stderr = stdout_bytes.decode("utf-8"), stderr_bytes.decode("utf-8") - if proc.returncode != 0: - print("EXECUTED COMMAND:", repr(command)) - print("RETURN CODE:", proc.returncode) - print() - print("STDOUT:") - print_offset(stdout) - print("STDERR:") - print_offset(stderr) - print() - - -Command = Callable[[], None] - - -def test(setup: Command, command: Command, teardown: Command) -> float: - setup() - start = time.time() - command() - end = time.time() - start - teardown() - return end - - -def make_touch_wrappers(filename: str) -> tuple[Command, Command]: - def setup() -> None: - execute(["touch", filename]) - - def teardown() -> None: - pass - - return setup, teardown - - -def make_change_wrappers(filename: str) -> tuple[Command, Command]: - copy: str | None = None - - def setup() -> None: - nonlocal copy - with open(filename) as stream: - copy = stream.read() - with open(filename, "a") as stream: - stream.write("\n\nfoo = 3") - - def teardown() -> None: - assert copy is not None - with open(filename, "w") as stream: - stream.write(copy) - - # Re-run to reset cache - execute(["python3", "-m", "mypy", "-i", "mypy"]), - - return setup, teardown - - -def main() -> None: - if len(sys.argv) != 2 or sys.argv[1] not in {"touch", "change"}: - print("First argument should be 'touch' or 'change'") - return - - if sys.argv[1] == "touch": - make_wrappers = make_touch_wrappers - verb = "Touching" - elif sys.argv[1] == "change": - make_wrappers = make_change_wrappers - verb = "Changing" - else: - raise AssertionError() - - print("Setting up...") - - baseline = test(lambda: None, lambda: execute(["python3", "-m", "mypy", "mypy"]), lambda: None) - print(f"Baseline: {baseline}") - - cold = test( - lambda: delete_folder(".mypy_cache"), - lambda: execute(["python3", "-m", "mypy", "-i", "mypy"]), - lambda: None, - ) - print(f"Cold cache: {cold}") - - warm = test( - lambda: None, lambda: execute(["python3", "-m", "mypy", "-i", "mypy"]), lambda: None - ) - print(f"Warm cache: {warm}") - - print() - - deltas = [] - for filename in glob.iglob("mypy/**/*.py", recursive=True): - print(f"{verb} {filename}") - - setup, teardown = make_wrappers(filename) - delta = test(setup, lambda: execute(["python3", "-m", "mypy", "-i", "mypy"]), teardown) - print(f" Time: {delta}") - deltas.append(delta) - print() - - print("Initial:") - print(f" Baseline: {baseline}") - print(f" Cold cache: {cold}") - print(f" Warm cache: {warm}") - print() - print("Aggregate:") - print(f" Times: {deltas}") - print(f" Mean: {statistics.mean(deltas)}") - print(f" Median: {statistics.median(deltas)}") - print(f" Stdev: {statistics.stdev(deltas)}") - print(f" Min: {min(deltas)}") - print(f" Max: {max(deltas)}") - print(f" Total: {sum(deltas)}") - print() - - -if __name__ == "__main__": - main() diff --git a/misc/upload-pypi.py b/misc/upload-pypi.py index 4d18b7d78ade..be8da9e44f86 100644 --- a/misc/upload-pypi.py +++ b/misc/upload-pypi.py @@ -32,12 +32,14 @@ def is_whl_or_tar(name: str) -> bool: def get_release_for_tag(tag: str) -> dict[str, Any]: with urlopen(f"{BASE}/{REPO}/releases/tags/{tag}") as f: data = json.load(f) + assert isinstance(data, dict) assert data["tag_name"] == tag return data def download_asset(asset: dict[str, Any], dst: Path) -> Path: name = asset["name"] + assert isinstance(name, str) download_url = asset["browser_download_url"] assert is_whl_or_tar(name) with urlopen(download_url) as src_file: @@ -47,7 +49,7 @@ def download_asset(asset: dict[str, Any], dst: Path) -> Path: def download_all_release_assets(release: dict[str, Any], dst: Path) -> None: - print(f"Downloading assets...") + print("Downloading assets...") with ThreadPoolExecutor() as e: for asset in e.map(lambda asset: download_asset(asset, dst), release["assets"]): print(f"Downloaded {asset}") @@ -68,7 +70,7 @@ def check_sdist(dist: Path, version: str) -> None: hashless_version = match.group(1) if match else version assert ( - f"'{hashless_version}'" in version_py_contents + f'"{hashless_version}"' in version_py_contents ), "Version does not match version.py in sdist" diff --git a/misc/variadics.py b/misc/variadics.py deleted file mode 100644 index c54e3fd8e30e..000000000000 --- a/misc/variadics.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Example of code generation approach to variadics. - -See https://github.com/python/typing/issues/193#issuecomment-236383893 -""" - -from __future__ import annotations - -LIMIT = 5 -BOUND = "object" - - -def prelude(limit: int, bound: str) -> None: - print("from typing import Callable, Iterable, Iterator, Tuple, TypeVar, overload") - print(f"Ts = TypeVar('Ts', bound={bound})") - print("R = TypeVar('R')") - for i in range(LIMIT): - print("T{i} = TypeVar('T{i}', bound={bound})".format(i=i + 1, bound=bound)) - - -def expand_template( - template: str, arg_template: str = "arg{i}: {Ts}", lower: int = 0, limit: int = LIMIT -) -> None: - print() - for i in range(lower, limit): - tvs = ", ".join(f"T{j+1}" for j in range(i)) - args = ", ".join(arg_template.format(i=j + 1, Ts=f"T{j+1}") for j in range(i)) - print("@overload") - s = template.format(Ts=tvs, argsTs=args) - s = s.replace("Tuple[]", "Tuple[()]") - print(s) - args_l = [arg_template.format(i=j + 1, Ts="Ts") for j in range(limit)] - args_l.append("*" + (arg_template.format(i="s", Ts="Ts"))) - args = ", ".join(args_l) - s = template.format(Ts="Ts, ...", argsTs=args) - s = s.replace("Callable[[Ts, ...]", "Callable[...") - print("@overload") - print(s) - - -def main(): - prelude(LIMIT, BOUND) - - # map() - expand_template("def map(func: Callable[[{Ts}], R], {argsTs}) -> R: ...", lower=1) - # zip() - expand_template("def zip({argsTs}) -> Tuple[{Ts}]: ...") - - # Naomi's examples - expand_template("def my_zip({argsTs}) -> Iterator[Tuple[{Ts}]]: ...", "arg{i}: Iterable[{Ts}]") - expand_template("def make_check({argsTs}) -> Callable[[{Ts}], bool]: ...") - expand_template( - "def my_map(f: Callable[[{Ts}], R], {argsTs}) -> Iterator[R]: ...", - "arg{i}: Iterable[{Ts}]", - ) - - -main() diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 1c372294383d..ee5fe5d295b8 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,3 +1,4 @@ +# NOTE: this needs to be kept in sync with the "requires" list in pyproject.toml typing_extensions>=3.10 mypy_extensions>=0.4.3 typed_ast>=1.4.0,<2; python_version<'3.8' diff --git a/mypy/api.py b/mypy/api.py index 18b92fe82064..589bfbbfa1a7 100644 --- a/mypy/api.py +++ b/mypy/api.py @@ -47,7 +47,7 @@ import sys from io import StringIO -from typing import Callable, TextIO +from typing import Callable, TextIO, cast def _run(main_wrapper: Callable[[TextIO, TextIO], None]) -> tuple[str, str, int]: @@ -59,7 +59,7 @@ def _run(main_wrapper: Callable[[TextIO, TextIO], None]) -> tuple[str, str, int] main_wrapper(stdout, stderr) exit_status = 0 except SystemExit as system_exit: - exit_status = system_exit.code + exit_status = cast(int, system_exit.code) return stdout.getvalue(), stderr.getvalue(), exit_status diff --git a/mypy/applytype.py b/mypy/applytype.py index b66e148ee0ab..1c401664568d 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -3,8 +3,8 @@ from typing import Callable, Sequence import mypy.subtypes -from mypy.expandtype import expand_type -from mypy.nodes import Context +from mypy.expandtype import expand_type, expand_unpack_with_variables +from mypy.nodes import ARG_POS, ARG_STAR, Context from mypy.types import ( AnyType, CallableType, @@ -16,6 +16,7 @@ TypeVarLikeType, TypeVarTupleType, TypeVarType, + UnpackType, get_proper_type, ) @@ -110,7 +111,33 @@ def apply_generic_arguments( callable = callable.expand_param_spec(nt) # Apply arguments to argument types. - arg_types = [expand_type(at, id_to_type) for at in callable.arg_types] + var_arg = callable.var_arg() + if var_arg is not None and isinstance(var_arg.typ, UnpackType): + expanded = expand_unpack_with_variables(var_arg.typ, id_to_type) + assert isinstance(expanded, list) + # Handle other cases later. + for t in expanded: + assert not isinstance(t, UnpackType) + star_index = callable.arg_kinds.index(ARG_STAR) + arg_kinds = ( + callable.arg_kinds[:star_index] + + [ARG_POS] * len(expanded) + + callable.arg_kinds[star_index + 1 :] + ) + arg_names = ( + callable.arg_names[:star_index] + + [None] * len(expanded) + + callable.arg_names[star_index + 1 :] + ) + arg_types = ( + [expand_type(at, id_to_type) for at in callable.arg_types[:star_index]] + + expanded + + [expand_type(at, id_to_type) for at in callable.arg_types[star_index + 1 :]] + ) + else: + arg_types = [expand_type(at, id_to_type) for at in callable.arg_types] + arg_kinds = callable.arg_kinds + arg_names = callable.arg_names # Apply arguments to TypeGuard if any. if callable.type_guard is not None: @@ -126,4 +153,6 @@ def apply_generic_arguments( ret_type=expand_type(callable.ret_type, id_to_type), variables=remaining_tvars, type_guard=type_guard, + arg_kinds=arg_kinds, + arg_names=arg_names, ) diff --git a/mypy/binder.py b/mypy/binder.py index 8e49f87c2506..d822aecec2f3 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -8,11 +8,21 @@ from mypy.erasetype import remove_instance_last_known_values from mypy.join import join_simple from mypy.literals import Key, literal, literal_hash, subkeys -from mypy.nodes import AssignmentExpr, Expression, IndexExpr, MemberExpr, NameExpr, RefExpr, Var +from mypy.nodes import Expression, IndexExpr, MemberExpr, NameExpr, RefExpr, TypeInfo, Var from mypy.subtypes import is_same_type, is_subtype -from mypy.types import AnyType, NoneType, PartialType, Type, TypeOfAny, UnionType, get_proper_type +from mypy.types import ( + AnyType, + NoneType, + PartialType, + Type, + TypeOfAny, + TypeType, + UnionType, + get_proper_type, +) +from mypy.typevars import fill_typevars_with_any -BindableExpression: _TypeAlias = Union[IndexExpr, MemberExpr, AssignmentExpr, NameExpr] +BindableExpression: _TypeAlias = Union[IndexExpr, MemberExpr, NameExpr] class Frame: @@ -133,7 +143,7 @@ def _get(self, key: Key, index: int = -1) -> Type | None: return None def put(self, expr: Expression, typ: Type) -> None: - if not isinstance(expr, (IndexExpr, MemberExpr, AssignmentExpr, NameExpr)): + if not isinstance(expr, (IndexExpr, MemberExpr, NameExpr)): return if not literal(expr): return @@ -439,8 +449,11 @@ def top_frame_context(self) -> Iterator[Frame]: def get_declaration(expr: BindableExpression) -> Type | None: - if isinstance(expr, RefExpr) and isinstance(expr.node, Var): - type = expr.node.type - if not isinstance(get_proper_type(type), PartialType): - return type + if isinstance(expr, RefExpr): + if isinstance(expr.node, Var): + type = expr.node.type + if not isinstance(get_proper_type(type), PartialType): + return type + elif isinstance(expr.node, TypeInfo): + return TypeType(fill_typevars_with_any(expr.node)) return None diff --git a/mypy/build.py b/mypy/build.py index 1d7ab25c989e..31851680ea82 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -35,7 +35,6 @@ Mapping, NamedTuple, NoReturn, - Optional, Sequence, TextIO, TypeVar, @@ -48,7 +47,9 @@ from mypy.checker import TypeChecker from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error from mypy.indirection import TypeIndirectionVisitor -from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable +from mypy.messages import MessageBuilder +from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable, TypeInfo +from mypy.partially_defined import PartiallyDefinedVariableVisitor from mypy.semanal import SemanticAnalyzer from mypy.semanal_pass1 import SemanticAnalyzerPreAnalysis from mypy.util import ( @@ -91,7 +92,12 @@ from mypy.plugins.default import DefaultPlugin from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor from mypy.stats import dump_type_stats -from mypy.stubinfo import is_legacy_bundled_package, legacy_bundled_packages +from mypy.stubinfo import ( + is_legacy_bundled_package, + legacy_bundled_packages, + non_bundled_packages, + stub_package_name, +) from mypy.types import Type from mypy.typestate import TypeState, reset_global_state from mypy.version import __version__ @@ -231,14 +237,13 @@ def _build( errors = Errors( options.show_error_context, options.show_column_numbers, - options.show_error_codes, + options.hide_error_codes, options.pretty, options.show_error_end, lambda path: read_py_file(path, cached_read), options.show_absolute_path, - options.enabled_error_codes, - options.disabled_error_codes, options.many_errors_threshold, + options, ) plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins) @@ -422,7 +427,7 @@ def plugin_error(message: str) -> NoReturn: errors.raise_error(use_stdout=False) custom_plugins: list[Plugin] = [] - errors.set_file(options.config_file, None) + errors.set_file(options.config_file, None, options) for plugin_path in options.plugins: func_name = "plugin" plugin_dir: str | None = None @@ -668,16 +673,10 @@ def __init__( raise CompileError( [f"Failed to find builtin module {module}, perhaps typeshed is broken?"] ) - if is_typeshed_file(path): - continue - if is_stub_package_file(path): + if is_typeshed_file(options.abs_custom_typeshed_dir, path) or is_stub_package_file( + path + ): continue - if options.custom_typeshed_dir is not None: - # Check if module lives under custom_typeshed_dir subtree - custom_typeshed_dir = os.path.abspath(options.custom_typeshed_dir) - path = os.path.abspath(path) - if os.path.commonpath((path, custom_typeshed_dir)) == custom_typeshed_dir: - continue raise CompileError( [ @@ -779,7 +778,7 @@ def correct_rel_imp(imp: ImportFrom | ImportAll) -> str: new_id = file_id + "." + imp.id if imp.id else file_id if not new_id: - self.errors.set_file(file.path, file.name) + self.errors.set_file(file.path, file.name, self.options) self.errors.report( imp.line, 0, "No parent module -- cannot perform relative import", blocker=True ) @@ -990,7 +989,7 @@ def write_deps_cache( error = True if error: - manager.errors.set_file(_cache_dir_prefix(manager.options), None) + manager.errors.set_file(_cache_dir_prefix(manager.options), None, manager.options) manager.errors.report(0, 0, "Error writing fine-grained dependencies cache", blocker=True) @@ -1054,7 +1053,7 @@ def generate_deps_for_cache(manager: BuildManager, graph: Graph) -> dict[str, di def write_plugins_snapshot(manager: BuildManager) -> None: """Write snapshot of versions and hashes of currently active plugins.""" if not manager.metastore.write(PLUGIN_SNAPSHOT_FILE, json.dumps(manager.plugins_snapshot)): - manager.errors.set_file(_cache_dir_prefix(manager.options), None) + manager.errors.set_file(_cache_dir_prefix(manager.options), None, manager.options) manager.errors.report(0, 0, "Error writing plugins snapshot", blocker=True) @@ -1125,6 +1124,7 @@ def read_deps_cache(manager: BuildManager, graph: Graph) -> dict[str, FgDepMeta] return None module_deps_metas = deps_meta["deps_meta"] + assert isinstance(module_deps_metas, dict) if not manager.options.skip_cache_mtime_checks: for id, meta in module_deps_metas.items(): try: @@ -1157,7 +1157,7 @@ def _load_json_file( result = json.loads(data) manager.add_stats(data_json_load_time=time.time() - t1) except json.JSONDecodeError: - manager.errors.set_file(file, None) + manager.errors.set_file(file, None, manager.options) manager.errors.report( -1, -1, @@ -1169,6 +1169,7 @@ def _load_json_file( ) return None else: + assert isinstance(result, dict) return result @@ -1292,11 +1293,15 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No ) # Don't check for path match, that is dealt with in validate_meta(). + # + # TODO: these `type: ignore`s wouldn't be necessary + # if the type annotations for CacheMeta were more accurate + # (all of these attributes can be `None`) if ( m.id != id - or m.mtime is None - or m.size is None - or m.dependencies is None + or m.mtime is None # type: ignore[redundant-expr] + or m.size is None # type: ignore[redundant-expr] + or m.dependencies is None # type: ignore[redundant-expr] or m.data_mtime is None ): manager.log(f"Metadata abandoned for {id}: attributes are missing") @@ -1396,8 +1401,8 @@ def validate_meta( st = manager.get_stat(path) except OSError: return None - if not (stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode)): - manager.log(f"Metadata abandoned for {id}: file {path} does not exist") + if not stat.S_ISDIR(st.st_mode) and not stat.S_ISREG(st.st_mode): + manager.log(f"Metadata abandoned for {id}: file or directory {path} does not exist") return None manager.add_stats(validate_stat_time=time.time() - t0) @@ -1935,6 +1940,8 @@ def __init__( raise if follow_imports == "silent": self.ignore_all = True + elif path and is_silent_import_module(manager, path): + self.ignore_all = True self.path = path if path: self.abspath = os.path.abspath(path) @@ -2206,7 +2213,7 @@ def parse_inline_configuration(self, source: str) -> None: if flags: changes, config_errors = parse_mypy_comments(flags, self.options) self.options = self.options.apply_changes(changes) - self.manager.errors.set_file(self.xpath, self.id) + self.manager.errors.set_file(self.xpath, self.id, self.options) for lineno, error in config_errors: self.manager.errors.report(lineno, 0, error) @@ -2331,6 +2338,17 @@ def type_check_second_pass(self) -> bool: self.time_spent_us += time_spent_us(t0) return result + def detect_partially_defined_vars(self, type_map: dict[Expression, Type]) -> None: + assert self.tree is not None, "Internal error: method must be called on parsed file only" + manager = self.manager + if manager.errors.is_error_code_enabled(codes.PARTIALLY_DEFINED): + manager.errors.set_file(self.xpath, self.tree.fullname, options=manager.options) + self.tree.accept( + PartiallyDefinedVariableVisitor( + MessageBuilder(manager.errors, manager.modules), type_map + ) + ) + def finish_passes(self) -> None: assert self.tree is not None, "Internal error: method must be called on parsed file only" manager = self.manager @@ -2345,7 +2363,24 @@ def finish_passes(self) -> None: # We should always patch indirect dependencies, even in full (non-incremental) builds, # because the cache still may be written, and it must be correct. - self._patch_indirect_dependencies(self.type_checker().module_refs, self.type_map()) + # TODO: find a more robust way to traverse *all* relevant types? + expr_types = set(self.type_map().values()) + symbol_types = set() + for _, sym, _ in self.tree.local_definitions(): + if sym.type is not None: + symbol_types.add(sym.type) + if isinstance(sym.node, TypeInfo): + # TypeInfo symbols have some extra relevant types. + symbol_types.update(sym.node.bases) + if sym.node.metaclass_type: + symbol_types.add(sym.node.metaclass_type) + if sym.node.typeddict_type: + symbol_types.add(sym.node.typeddict_type) + if sym.node.tuple_type: + symbol_types.add(sym.node.tuple_type) + self._patch_indirect_dependencies( + self.type_checker().module_refs, expr_types | symbol_types + ) if self.options.dump_inference_stats: dump_type_stats( @@ -2368,10 +2403,7 @@ def free_state(self) -> None: self._type_checker.reset() self._type_checker = None - def _patch_indirect_dependencies( - self, module_refs: set[str], type_map: dict[Expression, Type] - ) -> None: - types = set(type_map.values()) + def _patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None: assert None not in types valid = self.valid_references() @@ -2490,7 +2522,8 @@ def verify_dependencies(self, suppressed_only: bool = False) -> None: line = self.dep_line_map.get(dep, 1) try: if dep in self.ancestors: - state, ancestor = None, self # type: (Optional[State], Optional[State]) + state: State | None = None + ancestor: State | None = self else: state, ancestor = self, None # Called just for its side effects of producing diagnostics. @@ -2575,11 +2608,11 @@ def find_module_and_diagnose( if ( root_source # Honor top-level modules or ( - not result.endswith(".py") # Stubs are always normal - and not options.follow_imports_for_stubs - ) # except when they aren't - or id in mypy.semanal_main.core_modules - ): # core is always normal + result.endswith(".pyi") # Stubs are always normal + and not options.follow_imports_for_stubs # except when they aren't + ) + or id in mypy.semanal_main.core_modules # core is always normal + ): follow_imports = "normal" if skip_diagnose: pass @@ -2596,11 +2629,8 @@ def find_module_and_diagnose( else: skipping_module(manager, caller_line, caller_state, id, result) raise ModuleNotFound - if not manager.options.no_silence_site_packages: - for dir in manager.search_paths.package_path + manager.search_paths.typeshed_path: - if is_sub_path(result, dir): - # Silence errors in site-package dirs and typeshed - follow_imports = "silent" + if is_silent_import_module(manager, result): + follow_imports = "silent" return (result, follow_imports) else: # Could not find a module. Typically the reason is a @@ -2717,7 +2747,7 @@ def module_not_found( errors = manager.errors save_import_context = errors.import_context() errors.set_import_context(caller_state.import_context) - errors.set_file(caller_state.xpath, caller_state.id) + errors.set_file(caller_state.xpath, caller_state.id, caller_state.options) if target == "builtins": errors.report( line, 0, "Cannot find 'builtins' module. Typeshed appears broken!", blocker=True @@ -2726,17 +2756,16 @@ def module_not_found( else: daemon = manager.options.fine_grained_incremental msg, notes = reason.error_message_templates(daemon) - pyver = "%d.%d" % manager.options.python_version - errors.report(line, 0, msg.format(module=target, pyver=pyver), code=codes.IMPORT) + errors.report(line, 0, msg.format(module=target), code=codes.IMPORT) top_level, second_level = get_top_two_prefixes(target) - if second_level in legacy_bundled_packages: + if second_level in legacy_bundled_packages or second_level in non_bundled_packages: top_level = second_level for note in notes: if "{stub_dist}" in note: - note = note.format(stub_dist=legacy_bundled_packages[top_level]) + note = note.format(stub_dist=stub_package_name(top_level)) errors.report(line, 0, note, severity="note", only_once=True, code=codes.IMPORT) if reason is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: - manager.missing_stub_packages.add(legacy_bundled_packages[top_level]) + manager.missing_stub_packages.add(stub_package_name(top_level)) errors.set_import_context(save_import_context) @@ -2747,7 +2776,7 @@ def skipping_module( assert caller_state, (id, path) save_import_context = manager.errors.import_context() manager.errors.set_import_context(caller_state.import_context) - manager.errors.set_file(caller_state.xpath, caller_state.id) + manager.errors.set_file(caller_state.xpath, caller_state.id, manager.options) manager.errors.report(line, 0, f'Import of "{id}" ignored', severity="error") manager.errors.report( line, @@ -2766,7 +2795,7 @@ def skipping_ancestor(manager: BuildManager, id: str, path: str, ancestor_for: S # But beware, some package may be the ancestor of many modules, # so we'd need to cache the decision. manager.errors.set_import_context([]) - manager.errors.set_file(ancestor_for.xpath, ancestor_for.id) + manager.errors.set_file(ancestor_for.xpath, ancestor_for.id, manager.options) manager.errors.report( -1, -1, f'Ancestor package "{id}" ignored', severity="error", only_once=True ) @@ -3000,7 +3029,7 @@ def load_graph( except ModuleNotFound: continue if st.id in graph: - manager.errors.set_file(st.xpath, st.id) + manager.errors.set_file(st.xpath, st.id, manager.options) manager.errors.report( -1, -1, @@ -3359,6 +3388,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No graph[id].type_check_first_pass() if not graph[id].type_checker().deferred_nodes: unfinished_modules.discard(id) + graph[id].detect_partially_defined_vars(graph[id].type_map()) graph[id].finish_passes() while unfinished_modules: @@ -3367,6 +3397,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No continue if not graph[id].type_check_second_pass(): unfinished_modules.discard(id) + graph[id].detect_partially_defined_vars(graph[id].type_map()) graph[id].finish_passes() for id in stale: graph[id].generate_unused_ignore_notes() @@ -3542,3 +3573,12 @@ def record_missing_stub_packages(cache_dir: str, missing_stub_packages: set[str] else: if os.path.isfile(fnam): os.remove(fnam) + + +def is_silent_import_module(manager: BuildManager, path: str) -> bool: + if not manager.options.no_silence_site_packages: + for dir in manager.search_paths.package_path + manager.search_paths.typeshed_path: + if is_sub_path(path, dir): + # Silence errors in site-package dirs and typeshed + return True + return False diff --git a/mypy/checker.py b/mypy/checker.py index 4991177e6c1d..f478ce575722 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2,7 +2,6 @@ from __future__ import annotations -import fnmatch import itertools from collections import defaultdict from contextlib import contextmanager, nullcontext @@ -64,6 +63,7 @@ ARG_STAR, CONTRAVARIANT, COVARIANT, + FUNC_NO_INFO, GDEF, IMPLICITLY_ABSTRACT, INVARIANT, @@ -71,6 +71,7 @@ LDEF, LITERAL_TYPE, MDEF, + NOT_ABSTRACT, AssertStmt, AssignmentExpr, AssignmentStmt, @@ -115,6 +116,7 @@ ReturnStmt, StarExpr, Statement, + SymbolNode, SymbolTable, SymbolTableNode, TempNode, @@ -167,6 +169,7 @@ true_only, try_expanding_sum_type_to_union, try_getting_int_literals_from_type, + try_getting_str_literals, try_getting_str_literals_from_type, tuple_fallback, ) @@ -194,10 +197,12 @@ TypeTranslator, TypeType, TypeVarId, + TypeVarLikeType, TypeVarType, UnboundType, UninhabitedType, UnionType, + UnpackType, flatten_nested_unions, get_proper_type, get_proper_types, @@ -325,8 +330,6 @@ class TypeChecker(NodeVisitor[None], CheckerPluginInterface): current_node_deferred = False # Is this file a typeshed stub? is_typeshed_stub = False - # Should strict Optional-related errors be suppressed in this file? - suppress_none_errors = False # TODO: Get it from options instead options: Options # Used for collecting inferred attribute types so that they can be checked # for consistency. @@ -387,14 +390,9 @@ def __init__( self.pass_num = 0 self.current_node_deferred = False self.is_stub = tree.is_stub - self.is_typeshed_stub = is_typeshed_file(path) + self.is_typeshed_stub = is_typeshed_file(options.abs_custom_typeshed_dir, path) self.inferred_attribute_types = None - if options.strict_optional_whitelist is None: - self.suppress_none_errors = not options.show_none_errors - else: - self.suppress_none_errors = not any( - fnmatch.fnmatch(path, pattern) for pattern in options.strict_optional_whitelist - ) + # If True, process function definitions. If False, don't. This is used # for processing module top levels in fine-grained incremental mode. self.recurse_into_functions = True @@ -451,7 +449,9 @@ def check_first_pass(self) -> None: """ self.recurse_into_functions = True with state.strict_optional_set(self.options.strict_optional): - self.errors.set_file(self.path, self.tree.fullname, scope=self.tscope) + self.errors.set_file( + self.path, self.tree.fullname, scope=self.tscope, options=self.options + ) with self.tscope.module_scope(self.tree.fullname): with self.enter_partial_types(), self.binder.top_frame_context(): for d in self.tree.defs: @@ -490,7 +490,9 @@ def check_second_pass( with state.strict_optional_set(self.options.strict_optional): if not todo and not self.deferred_nodes: return False - self.errors.set_file(self.path, self.tree.fullname, scope=self.tscope) + self.errors.set_file( + self.path, self.tree.fullname, scope=self.tscope, options=self.options + ) with self.tscope.module_scope(self.tree.fullname): self.pass_num += 1 if not todo: @@ -621,7 +623,7 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: self.visit_decorator(cast(Decorator, defn.items[0])) for fdef in defn.items: assert isinstance(fdef, Decorator) - self.check_func_item(fdef.func, name=fdef.func.name) + self.check_func_item(fdef.func, name=fdef.func.name, allow_empty=True) if fdef.func.abstract_status in (IS_ABSTRACT, IMPLICITLY_ABSTRACT): num_abstract += 1 if num_abstract not in (0, len(defn.items)): @@ -728,9 +730,10 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: # This is to match the direction the implementation's return # needs to be compatible in. if impl_type.variables: - impl = unify_generic_callable( - impl_type, - sig1, + impl: CallableType | None = unify_generic_callable( + # Normalize both before unifying + impl_type.with_unpacked_kwargs(), + sig1.with_unpacked_kwargs(), ignore_return=False, return_constraint_direction=SUPERTYPE_OF, ) @@ -747,14 +750,14 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: # Is the overload alternative's arguments subtypes of the implementation's? if not is_callable_compatible( - impl, sig1, is_compat=is_subtype_no_promote, ignore_return=True + impl, sig1, is_compat=is_subtype, ignore_return=True ): self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl) # Is the overload alternative's return type a subtype of the implementation's? if not ( - is_subtype_no_promote(sig1.ret_type, impl.ret_type) - or is_subtype_no_promote(impl.ret_type, sig1.ret_type) + is_subtype(sig1.ret_type, impl.ret_type) + or is_subtype(impl.ret_type, sig1.ret_type) ): self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl) @@ -950,17 +953,14 @@ def _visit_func_def(self, defn: FuncDef) -> None: new_type = self.function_type(defn) if isinstance(defn.original_def, FuncDef): # Function definition overrides function definition. - if not is_same_type(new_type, self.function_type(defn.original_def)): - self.msg.incompatible_conditional_function_def(defn) + old_type = self.function_type(defn.original_def) + if not is_same_type(new_type, old_type): + self.msg.incompatible_conditional_function_def(defn, old_type, new_type) else: # Function definition overrides a variable initialized via assignment or a # decorated function. orig_type = defn.original_def.type - if orig_type is None: - # XXX This can be None, as happens in - # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse - self.msg.note("Internal mypy error checking function redefinition", defn) - return + assert orig_type is not None, f"Error checking function redefinition {defn}" if isinstance(orig_type, PartialType): if orig_type.type is None: # Ah this is a partial type. Give it the type of the function. @@ -977,7 +977,9 @@ def _visit_func_def(self, defn: FuncDef) -> None: # Trying to redefine something like partial empty list as function. self.fail(message_registry.INCOMPATIBLE_REDEFINITION, defn) else: - # TODO: Update conditional type binder. + name_expr = NameExpr(defn.name) + name_expr.node = defn.original_def + self.binder.assign_type(name_expr, new_type, orig_type) self.check_subtype( new_type, orig_type, @@ -988,7 +990,11 @@ def _visit_func_def(self, defn: FuncDef) -> None: ) def check_func_item( - self, defn: FuncItem, type_override: CallableType | None = None, name: str | None = None + self, + defn: FuncItem, + type_override: CallableType | None = None, + name: str | None = None, + allow_empty: bool = False, ) -> None: """Type check a function. @@ -1002,7 +1008,7 @@ def check_func_item( typ = type_override.copy_modified(line=typ.line, column=typ.column) if isinstance(typ, CallableType): with self.enter_attribute_inference_context(): - self.check_func_def(defn, typ, name) + self.check_func_def(defn, typ, name, allow_empty) else: raise RuntimeError("Not supported") @@ -1019,7 +1025,9 @@ def enter_attribute_inference_context(self) -> Iterator[None]: yield None self.inferred_attribute_types = old_types - def check_func_def(self, defn: FuncItem, typ: CallableType, name: str | None) -> None: + def check_func_def( + self, defn: FuncItem, typ: CallableType, name: str | None, allow_empty: bool = False + ) -> None: """Type check a function definition.""" # Expand type variables with value restrictions to ordinary types. expanded = self.expand_typevars(defn, typ) @@ -1034,11 +1042,13 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str | None) -> # precise type. if isinstance(item, FuncDef): fdef = item - # Check if __init__ has an invalid, non-None return type. + # Check if __init__ has an invalid return type. if ( fdef.info and fdef.name in ("__init__", "__init_subclass__") - and not isinstance(get_proper_type(typ.ret_type), NoneType) + and not isinstance( + get_proper_type(typ.ret_type), (NoneType, UninhabitedType) + ) and not self.dynamic_funcs[-1] ): self.fail( @@ -1161,11 +1171,20 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str | None) -> ctx = typ self.fail(message_registry.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, ctx) if typ.arg_kinds[i] == nodes.ARG_STAR: - if not isinstance(arg_type, ParamSpecType): + if isinstance(arg_type, ParamSpecType): + pass + elif isinstance(arg_type, UnpackType): + arg_type = TupleType( + [arg_type], + fallback=self.named_generic_type( + "builtins.tuple", [self.named_type("builtins.object")] + ), + ) + else: # builtins.tuple[T] is typing.Tuple[T, ...] arg_type = self.named_generic_type("builtins.tuple", [arg_type]) elif typ.arg_kinds[i] == nodes.ARG_STAR2: - if not isinstance(arg_type, ParamSpecType): + if not isinstance(arg_type, ParamSpecType) and not typ.unpack_kwargs: arg_type = self.named_generic_type( "builtins.dict", [self.str_type(), arg_type] ) @@ -1189,7 +1208,7 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str | None) -> self.accept(item.body) unreachable = self.binder.is_unreachable() - if not unreachable and not body_is_trivial: + if not unreachable: if defn.is_generator or is_named_instance( self.return_types[-1], "typing.AwaitableGenerator" ): @@ -1202,27 +1221,79 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str | None) -> return_type = self.return_types[-1] return_type = get_proper_type(return_type) + allow_empty = allow_empty or self.options.allow_empty_bodies + + show_error = ( + not body_is_trivial + or + # Allow empty bodies for abstract methods, overloads, in tests and stubs. + ( + not allow_empty + and not ( + isinstance(defn, FuncDef) and defn.abstract_status != NOT_ABSTRACT + ) + and not self.is_stub + ) + ) + + # Ignore plugin generated methods, these usually don't need any bodies. + if defn.info is not FUNC_NO_INFO and ( + defn.name not in defn.info.names or defn.info.names[defn.name].plugin_generated + ): + show_error = False + + # Ignore also definitions that appear in `if TYPE_CHECKING: ...` blocks. + # These can't be called at runtime anyway (similar to plugin-generated). + if isinstance(defn, FuncDef) and defn.is_mypy_only: + show_error = False + + # We want to minimize the fallout from checking empty bodies + # that was absent in many mypy versions. + if body_is_trivial and is_subtype(NoneType(), return_type): + show_error = False + + may_be_abstract = ( + body_is_trivial + and defn.info is not FUNC_NO_INFO + and defn.info.metaclass_type is not None + and defn.info.metaclass_type.type.has_base("abc.ABCMeta") + ) + if self.options.warn_no_return: - if not isinstance(return_type, (NoneType, AnyType)): + if ( + not self.current_node_deferred + and not isinstance(return_type, (NoneType, AnyType)) + and show_error + ): # Control flow fell off the end of a function that was - # declared to return a non-None type and is not - # entirely pass/Ellipsis/raise NotImplementedError. + # declared to return a non-None type. if isinstance(return_type, UninhabitedType): # This is a NoReturn function - self.fail(message_registry.INVALID_IMPLICIT_RETURN, defn) + msg = message_registry.INVALID_IMPLICIT_RETURN else: - self.fail(message_registry.MISSING_RETURN_STATEMENT, defn) - else: + msg = message_registry.MISSING_RETURN_STATEMENT + if body_is_trivial: + msg = msg._replace(code=codes.EMPTY_BODY) + self.fail(msg, defn) + if may_be_abstract: + self.note(message_registry.EMPTY_BODY_ABSTRACT, defn) + elif show_error: + msg = message_registry.INCOMPATIBLE_RETURN_VALUE_TYPE + if body_is_trivial: + msg = msg._replace(code=codes.EMPTY_BODY) # similar to code in check_return_stmt - self.check_subtype( - subtype_label="implicitly returns", - subtype=NoneType(), - supertype_label="expected", - supertype=return_type, - context=defn, - msg=message_registry.INCOMPATIBLE_RETURN_VALUE_TYPE, - code=codes.RETURN_VALUE, - ) + if ( + not self.check_subtype( + subtype_label="implicitly returns", + subtype=NoneType(), + supertype_label="expected", + supertype=return_type, + context=defn, + msg=msg, + ) + and may_be_abstract + ): + self.note(message_registry.EMPTY_BODY_ABSTRACT, defn) self.return_types.pop() @@ -1230,13 +1301,23 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str | None) -> def check_unbound_return_typevar(self, typ: CallableType) -> None: """Fails when the return typevar is not defined in arguments.""" - if typ.ret_type in typ.variables: - arg_type_visitor = CollectArgTypes() + if isinstance(typ.ret_type, TypeVarType) and typ.ret_type in typ.variables: + arg_type_visitor = CollectArgTypeVarTypes() for argtype in typ.arg_types: argtype.accept(arg_type_visitor) if typ.ret_type not in arg_type_visitor.arg_types: self.fail(message_registry.UNBOUND_TYPEVAR, typ.ret_type, code=TYPE_VAR) + upper_bound = get_proper_type(typ.ret_type.upper_bound) + if not ( + isinstance(upper_bound, Instance) + and upper_bound.type.fullname == "builtins.object" + ): + self.note( + "Consider using the upper bound " + f"{format_type(typ.ret_type.upper_bound)} instead", + context=typ.ret_type, + ) def check_default_args(self, item: FuncItem, body_is_trivial: bool) -> None: for arg in item.arguments: @@ -1250,14 +1331,27 @@ def check_default_args(self, item: FuncItem, body_is_trivial: bool) -> None: msg += f"tuple argument {name[12:]}" else: msg += f'argument "{name}"' + if ( + not self.options.implicit_optional + and isinstance(arg.initializer, NameExpr) + and arg.initializer.fullname == "builtins.None" + ): + notes = [ + "PEP 484 prohibits implicit Optional. " + "Accordingly, mypy has changed its default to no_implicit_optional=True", + "Use https://github.com/hauntsaninja/no_implicit_optional to automatically " + "upgrade your codebase", + ] + else: + notes = None self.check_simple_assignment( arg.variable.type, arg.initializer, context=arg.initializer, - msg=msg, + msg=ErrorMessage(msg, code=codes.ASSIGNMENT), lvalue_name="argument", rvalue_name="default", - code=codes.ASSIGNMENT, + notes=notes, ) def is_forward_op_method(self, method_name: str) -> bool: @@ -1314,7 +1408,7 @@ def check___new___signature(self, fdef: FuncDef, typ: CallableType) -> None: bound_type = bind_self(typ, self_type, is_classmethod=True) # Check that __new__ (after binding cls) returns an instance # type (or any). - if isinstance(fdef.info, TypeInfo) and fdef.info.is_metaclass(): + if fdef.info.is_metaclass(): # This is a metaclass, so it must return a new unrelated type. self.check_subtype( bound_type.ret_type, @@ -1324,7 +1418,9 @@ def check___new___signature(self, fdef: FuncDef, typ: CallableType) -> None: "returns", "but must return a subtype of", ) - elif not isinstance(get_proper_type(bound_type.ret_type), (AnyType, Instance, TupleType)): + elif not isinstance( + get_proper_type(bound_type.ret_type), (AnyType, Instance, TupleType, UninhabitedType) + ): self.fail( message_registry.NON_INSTANCE_NEW_TYPE.format(format_type(bound_type.ret_type)), fdef, @@ -1617,6 +1713,8 @@ def check_slots_definition(self, typ: Type, context: Context) -> None: def check_match_args(self, var: Var, typ: Type, context: Context) -> None: """Check that __match_args__ contains literal strings""" + if not self.scope.active_class(): + return typ = get_proper_type(typ) if not isinstance(typ, TupleType) or not all( [is_string_literal(item) for item in typ.items] @@ -1719,6 +1817,7 @@ def check_method_override_for_base_with_name( context = defn.func # Construct the type of the overriding method. + # TODO: this logic is much less complete than similar one in checkmember.py if isinstance(defn, (FuncDef, OverloadedFuncDef)): typ: Type = self.function_type(defn) override_class_or_static = defn.is_class or defn.is_static @@ -1768,15 +1867,37 @@ def check_method_override_for_base_with_name( original_class_or_static = fdef.is_class or fdef.is_static else: original_class_or_static = False # a variable can't be class or static + + if isinstance(original_type, FunctionLike): + original_type = self.bind_and_map_method(base_attr, original_type, defn.info, base) + if original_node and is_property(original_node): + original_type = get_property_type(original_type) + + if isinstance(typ, FunctionLike) and is_property(defn): + typ = get_property_type(typ) + if ( + isinstance(original_node, Var) + and not original_node.is_final + and (not original_node.is_property or original_node.is_settable_property) + and isinstance(defn, Decorator) + ): + # We only give an error where no other similar errors will be given. + if not isinstance(original_type, AnyType): + self.msg.fail( + "Cannot override writeable attribute with read-only property", + # Give an error on function line to match old behaviour. + defn.func, + code=codes.OVERRIDE, + ) + if isinstance(original_type, AnyType) or isinstance(typ, AnyType): pass elif isinstance(original_type, FunctionLike) and isinstance(typ, FunctionLike): - original = self.bind_and_map_method(base_attr, original_type, defn.info, base) # Check that the types are compatible. # TODO overloaded signatures self.check_override( typ, - original, + original_type, defn.name, name, base.name, @@ -1791,8 +1912,8 @@ def check_method_override_for_base_with_name( # pass elif ( - base_attr.node - and not self.is_writable_attribute(base_attr.node) + original_node + and not self.is_writable_attribute(original_node) and is_subtype(typ, original_type) ): # If the attribute is read-only, allow covariance @@ -1875,7 +1996,7 @@ def check_override( ): fail = True op_method_wider_note = True - if isinstance(original, FunctionLike) and isinstance(override, FunctionLike): + if isinstance(override, FunctionLike): if original_class_or_static and not override_class_or_static: fail = True elif isinstance(original, CallableType) and isinstance(override, CallableType): @@ -1887,6 +2008,13 @@ def check_override( if fail: emitted_msg = False + + # Normalize signatures, so we get better diagnostics. + if isinstance(override, (CallableType, Overloaded)): + override = override.with_unpacked_kwargs() + if isinstance(original, (CallableType, Overloaded)): + original = original.with_unpacked_kwargs() + if ( isinstance(override, CallableType) and isinstance(original, CallableType) @@ -2009,6 +2137,7 @@ def visit_class_def(self, defn: ClassDef) -> None: if not defn.has_incompatible_baseclass: # Otherwise we've already found errors; more errors are not useful self.check_multiple_inheritance(typ) + self.check_metaclass_compatibility(typ) self.check_final_deletable(typ) if defn.decorators: @@ -2037,6 +2166,24 @@ def visit_class_def(self, defn: ClassDef) -> None: self.allow_abstract_call = old_allow_abstract_call # TODO: Apply the sig to the actual TypeInfo so we can handle decorators # that completely swap out the type. (e.g. Callable[[Type[A]], Type[B]]) + if typ.defn.type_vars: + for base_inst in typ.bases: + for base_tvar, base_decl_tvar in zip( + base_inst.args, base_inst.type.defn.type_vars + ): + if ( + isinstance(base_tvar, TypeVarType) + and base_tvar.variance != INVARIANT + and isinstance(base_decl_tvar, TypeVarType) + and base_decl_tvar.variance != base_tvar.variance + ): + self.fail( + f'Variance of TypeVar "{base_tvar.name}" incompatible ' + "with variance in parent type", + context=defn, + code=codes.TYPE_VAR, + ) + if typ.is_protocol and typ.defn.type_vars: self.check_protocol_variance(defn) if not defn.has_incompatible_baseclass and defn.info.is_enum: @@ -2254,18 +2401,27 @@ def check_multiple_inheritance(self, typ: TypeInfo) -> None: if name in base2.names and base2 not in base.mro: self.check_compatibility(name, base, base2, typ) - def determine_type_of_class_member(self, sym: SymbolTableNode) -> Type | None: + def determine_type_of_member(self, sym: SymbolTableNode) -> Type | None: if sym.type is not None: return sym.type if isinstance(sym.node, FuncBase): return self.function_type(sym.node) if isinstance(sym.node, TypeInfo): - # nested class - return type_object_type(sym.node, self.named_type) + if sym.node.typeddict_type: + # We special-case TypedDict, because they don't define any constructor. + return self.expr_checker.typeddict_callable(sym.node) + else: + return type_object_type(sym.node, self.named_type) if isinstance(sym.node, TypeVarExpr): # Use of TypeVars is rejected in an expression/runtime context, so # we don't need to check supertype compatibility for them. return AnyType(TypeOfAny.special_form) + if isinstance(sym.node, TypeAlias): + with self.msg.filter_errors(): + # Suppress any errors, they will be given when analyzing the corresponding node. + # Here we may have incorrect options and location context. + return self.expr_checker.alias_type_in_runtime_context(sym.node, ctx=sym.node) + # TODO: handle more node kinds here. return None def check_compatibility( @@ -2296,8 +2452,8 @@ class C(B, A[int]): ... # this is unsafe because... return first = base1.names[name] second = base2.names[name] - first_type = get_proper_type(self.determine_type_of_class_member(first)) - second_type = get_proper_type(self.determine_type_of_class_member(second)) + first_type = get_proper_type(self.determine_type_of_member(first)) + second_type = get_proper_type(self.determine_type_of_member(second)) if isinstance(first_type, FunctionLike) and isinstance(second_type, FunctionLike): if first_type.is_type_obj() and second_type.is_type_obj(): @@ -2337,6 +2493,35 @@ class C(B, A[int]): ... # this is unsafe because... if not ok: self.msg.base_class_definitions_incompatible(name, base1, base2, ctx) + def check_metaclass_compatibility(self, typ: TypeInfo) -> None: + """Ensures that metaclasses of all parent types are compatible.""" + if ( + typ.is_metaclass() + or typ.is_protocol + or typ.is_named_tuple + or typ.is_enum + or typ.typeddict_type is not None + ): + return # Reasonable exceptions from this check + + metaclasses = [ + entry.metaclass_type + for entry in typ.mro[1:-1] + if entry.metaclass_type + and not is_named_instance(entry.metaclass_type, "builtins.type") + ] + if not metaclasses: + return + if typ.metaclass_type is not None and all( + is_subtype(typ.metaclass_type, meta) for meta in metaclasses + ): + return + self.fail( + "Metaclass conflict: the metaclass of a derived class must be " + "a (non-strict) subclass of the metaclasses of all its bases", + typ, + ) + def visit_import_from(self, node: ImportFrom) -> None: self.check_import(node) @@ -2353,8 +2538,8 @@ def check_import(self, node: ImportBase) -> None: if lvalue_type is None: # TODO: This is broken. lvalue_type = AnyType(TypeOfAny.special_form) - message = '{} "{}"'.format( - message_registry.INCOMPATIBLE_IMPORT_OF, cast(NameExpr, assign.rvalue).name + message = message_registry.INCOMPATIBLE_IMPORT_OF.format( + cast(NameExpr, assign.rvalue).name ) self.check_simple_assignment( lvalue_type, @@ -2387,6 +2572,7 @@ def should_report_unreachable_issues(self) -> bool: return ( self.in_checked_function() and self.options.warn_unreachable + and not self.current_node_deferred and not self.binder.is_unreachable_warning_suppressed() ) @@ -2467,6 +2653,9 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: ): self.fail(message_registry.DEPENDENT_FINAL_IN_CLASS_BODY, s) + if s.unanalyzed_type and not self.in_checked_function(): + self.msg.annotation_in_unchecked_function(context=s) + def check_type_alias_rvalue(self, s: AssignmentStmt) -> None: if not (self.is_stub and isinstance(s.rvalue, OpExpr) and s.rvalue.op == "|"): # We do this mostly for compatibility with old semantic analyzer. @@ -2617,9 +2806,7 @@ def check_assignment( lvalue_type = make_optional_type(lvalue_type) self.set_inferred_type(lvalue.node, lvalue, lvalue_type) - rvalue_type = self.check_simple_assignment( - lvalue_type, rvalue, context=rvalue, code=codes.ASSIGNMENT - ) + rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, context=rvalue) # Special case: only non-abstract non-protocol classes can be assigned to # variables with explicit type Type[A], where A is protocol or abstract. @@ -2649,7 +2836,8 @@ def check_assignment( self.check_indexed_assignment(index_lvalue, rvalue, lvalue) if inferred: - rvalue_type = self.expr_checker.accept(rvalue) + type_context = self.get_variable_type_context(inferred) + rvalue_type = self.expr_checker.accept(rvalue, type_context=type_context) if not ( inferred.is_final or (isinstance(lvalue, NameExpr) and lvalue.name == "__match_args__") @@ -2661,6 +2849,29 @@ def check_assignment( # (type, operator) tuples for augmented assignments supported with partial types partial_type_augmented_ops: Final = {("builtins.list", "+"), ("builtins.set", "|")} + def get_variable_type_context(self, inferred: Var) -> Type | None: + type_contexts = [] + if inferred.info: + for base in inferred.info.mro[1:]: + base_type, base_node = self.lvalue_type_from_base(inferred, base) + if ( + base_type + and not (isinstance(base_node, Var) and base_node.invalid_partial_type) + and not isinstance(base_type, PartialType) + ): + type_contexts.append(base_type) + # Use most derived supertype as type context if available. + if not type_contexts: + return None + candidate = type_contexts[0] + for other in type_contexts: + if is_proper_subtype(other, candidate): + candidate = other + elif not is_subtype(candidate, other): + # Multiple incompatible candidates, cannot use any of them as context. + return None + return candidate + def try_infer_partial_generic_type_from_assignment( self, lvalue: Lvalue, rvalue: Expression, op: str ) -> None: @@ -2736,12 +2947,8 @@ def check_compatibility_all_supers( # The type of "__slots__" and some other attributes usually doesn't need to # be compatible with a base class. We'll still check the type of "__slots__" # against "object" as an exception. - if ( - isinstance(lvalue_node, Var) - and lvalue_node.allow_incompatible_override - and not ( - lvalue_node.name == "__slots__" and base.fullname == "builtins.object" - ) + if lvalue_node.allow_incompatible_override and not ( + lvalue_node.name == "__slots__" and base.fullname == "builtins.object" ): continue @@ -2749,6 +2956,8 @@ def check_compatibility_all_supers( continue base_type, base_node = self.lvalue_type_from_base(lvalue_node, base) + if isinstance(base_type, PartialType): + base_type = None if base_type: assert base_node is not None @@ -2826,7 +3035,6 @@ def check_compatibility_super( message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, "expression has type", f'base class "{base.name}" defined the type as', - code=codes.ASSIGNMENT, ) return True @@ -3161,6 +3369,9 @@ def check_multi_assignment( # TODO: maybe elsewhere; redundant. rvalue_type = get_proper_type(rv_type or self.expr_checker.accept(rvalue)) + if isinstance(rvalue_type, TypeVarLikeType): + rvalue_type = get_proper_type(rvalue_type.upper_bound) + if isinstance(rvalue_type, UnionType): # If this is an Optional type in non-strict Optional code, unwrap it. relevant_items = rvalue_type.relevant_items() @@ -3611,11 +3822,11 @@ def check_simple_assignment( lvalue_type: Type | None, rvalue: Expression, context: Context, - msg: str = message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, + msg: ErrorMessage = message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, lvalue_name: str = "variable", rvalue_name: str = "expression", *, - code: ErrorCode | None = None, + notes: list[str] | None = None, ) -> Type: if self.is_stub and isinstance(rvalue, EllipsisExpr): # '...' is always a valid initializer in a stub. @@ -3640,7 +3851,7 @@ def check_simple_assignment( msg, f"{rvalue_name} has type", f"{lvalue_name} has type", - code=code, + notes=notes, ) return rvalue_type @@ -3664,16 +3875,12 @@ def check_member_assignment( if (isinstance(instance_type, FunctionLike) and instance_type.is_type_obj()) or isinstance( instance_type, TypeType ): - rvalue_type = self.check_simple_assignment( - attribute_type, rvalue, context, code=codes.ASSIGNMENT - ) + rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context) return rvalue_type, attribute_type, True if not isinstance(attribute_type, Instance): # TODO: support __set__() for union types. - rvalue_type = self.check_simple_assignment( - attribute_type, rvalue, context, code=codes.ASSIGNMENT - ) + rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context) return rvalue_type, attribute_type, True mx = MemberContext( @@ -3692,9 +3899,7 @@ def check_member_assignment( # the return type of __get__. This doesn't match the python semantics, # (which allow you to override the descriptor with any value), but preserves # the type of accessing the attribute (even after the override). - rvalue_type = self.check_simple_assignment( - get_type, rvalue, context, code=codes.ASSIGNMENT - ) + rvalue_type = self.check_simple_assignment(get_type, rvalue, context) return rvalue_type, get_type, True dunder_set = attribute_type.type.get_method("__set__") @@ -3761,9 +3966,7 @@ def check_member_assignment( # and '__get__' type is narrower than '__set__', then we invoke the binder to narrow type # by this assignment. Technically, this is not safe, but in practice this is # what a user expects. - rvalue_type = self.check_simple_assignment( - set_type, rvalue, context, code=codes.ASSIGNMENT - ) + rvalue_type = self.check_simple_assignment(set_type, rvalue, context) infer = is_subtype(rvalue_type, get_type) and is_subtype(get_type, set_type) return rvalue_type if infer else set_type, get_type, infer @@ -3937,7 +4140,6 @@ def check_return_stmt(self, s: ReturnStmt) -> None: context=s.expr, outer_context=s, msg=message_registry.INCOMPATIBLE_RETURN_VALUE_TYPE, - code=codes.RETURN_VALUE, ) else: # Empty returns are valid in Generators with Any typed returns, but not in @@ -4105,20 +4307,13 @@ def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: with self.binder.frame_context(can_skip=True, fall_through=4): typ = s.types[i] if typ: - t = self.check_except_handler_test(typ) + t = self.check_except_handler_test(typ, s.is_star) var = s.vars[i] if var: # To support local variables, we make this a definition line, # causing assignment to set the variable's type. var.is_inferred_def = True - # We also temporarily set current_node_deferred to False to - # make sure the inference happens. - # TODO: Use a better solution, e.g. a - # separate Var for each except block. - am_deferring = self.current_node_deferred - self.current_node_deferred = False self.check_assignment(var, self.temp_node(t, var)) - self.current_node_deferred = am_deferring self.accept(s.handlers[i]) var = s.vars[i] if var: @@ -4132,7 +4327,7 @@ def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: if s.else_body: self.accept(s.else_body) - def check_except_handler_test(self, n: Expression) -> Type: + def check_except_handler_test(self, n: Expression, is_star: bool) -> Type: """Type check an exception handler test clause.""" typ = self.expr_checker.accept(n) @@ -4148,22 +4343,47 @@ def check_except_handler_test(self, n: Expression) -> Type: item = ttype.items[0] if not item.is_type_obj(): self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) - return AnyType(TypeOfAny.from_error) - exc_type = item.ret_type + return self.default_exception_type(is_star) + exc_type = erase_typevars(item.ret_type) elif isinstance(ttype, TypeType): exc_type = ttype.item else: self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) - return AnyType(TypeOfAny.from_error) + return self.default_exception_type(is_star) if not is_subtype(exc_type, self.named_type("builtins.BaseException")): self.fail(message_registry.INVALID_EXCEPTION_TYPE, n) - return AnyType(TypeOfAny.from_error) + return self.default_exception_type(is_star) all_types.append(exc_type) + if is_star: + new_all_types: list[Type] = [] + for typ in all_types: + if is_proper_subtype(typ, self.named_type("builtins.BaseExceptionGroup")): + self.fail(message_registry.INVALID_EXCEPTION_GROUP, n) + new_all_types.append(AnyType(TypeOfAny.from_error)) + else: + new_all_types.append(typ) + return self.wrap_exception_group(new_all_types) return make_simplified_union(all_types) + def default_exception_type(self, is_star: bool) -> Type: + """Exception type to return in case of a previous type error.""" + any_type = AnyType(TypeOfAny.from_error) + if is_star: + return self.named_generic_type("builtins.ExceptionGroup", [any_type]) + return any_type + + def wrap_exception_group(self, types: Sequence[Type]) -> Type: + """Transform except* variable type into an appropriate exception group.""" + arg = make_simplified_union(types) + if is_subtype(arg, self.named_type("builtins.Exception")): + base = "builtins.ExceptionGroup" + else: + base = "builtins.BaseExceptionGroup" + return self.named_generic_type(base, [arg]) + def get_types_from_except_handler(self, typ: Type, n: Expression) -> list[Type]: """Helper for check_except_handler_test to retrieve handler types.""" typ = get_proper_type(typ) @@ -4175,7 +4395,7 @@ def get_types_from_except_handler(self, typ: Type, n: Expression) -> list[Type]: for item in typ.relevant_items() for union_typ in self.get_types_from_except_handler(item, n) ] - elif isinstance(typ, Instance) and is_named_instance(typ, "builtins.tuple"): + elif is_named_instance(typ, "builtins.tuple"): # variadic tuple return [typ.args[0]] else: @@ -4209,6 +4429,10 @@ def analyze_iterable_item_type(self, expr: Expression) -> tuple[Type, Type]: iterable = get_proper_type(echk.accept(expr)) iterator = echk.check_method_call_by_name("__iter__", iterable, [], [], expr)[0] + int_type = self.analyze_range_native_int_type(expr) + if int_type: + return iterator, int_type + if isinstance(iterable, TupleType): joined: Type = UninhabitedType() for item in iterable.items: @@ -4218,6 +4442,36 @@ def analyze_iterable_item_type(self, expr: Expression) -> tuple[Type, Type]: # Non-tuple iterable. return iterator, echk.check_method_call_by_name("__next__", iterator, [], [], expr)[0] + def analyze_range_native_int_type(self, expr: Expression) -> Type | None: + """Try to infer native int item type from arguments to range(...). + + For example, return i64 if the expression is "range(0, i64(n))". + + Return None if unsuccessful. + """ + if ( + isinstance(expr, CallExpr) + and isinstance(expr.callee, RefExpr) + and expr.callee.fullname == "builtins.range" + and 1 <= len(expr.args) <= 3 + and all(kind == ARG_POS for kind in expr.arg_kinds) + ): + native_int: Type | None = None + ok = True + for arg in expr.args: + argt = get_proper_type(self.lookup_type(arg)) + if isinstance(argt, Instance) and argt.type.fullname in ( + "mypy_extensions.i64", + "mypy_extensions.i32", + ): + if native_int is None: + native_int = argt + elif argt != native_int: + ok = False + if ok and native_int: + return native_int + return None + def analyze_container_item_type(self, typ: Type) -> Type | None: """Check if a type is a nominal container of a union of such. @@ -4307,7 +4561,8 @@ def visit_decorator(self, e: Decorator) -> None: if len([k for k in sig.arg_kinds if k.is_required()]) > 1: self.msg.fail("Too many arguments for property", e) self.check_incompatible_property_override(e) - if e.func.info and not e.func.is_dynamic(): + # For overloaded functions we already checked override for overload as a whole. + if e.func.info and not e.func.is_dynamic() and not e.is_overload: self.check_method_override(e) if e.func.info and e.func.name in ("__init__", "__new__"): @@ -4544,7 +4799,7 @@ def make_fake_typeinfo( cdef.info = info info.bases = bases calculate_mro(info) - info.calculate_metaclass_type() + info.metaclass_type = info.calculate_metaclass_type() return cdef, info def intersect_instances( @@ -4629,7 +4884,7 @@ def _make_fake_typeinfo_and_full_name( return None curr_module.names[full_name] = SymbolTableNode(GDEF, info) - return Instance(info, []) + return Instance(info, [], extra_attrs=instances[0].extra_attrs or instances[1].extra_attrs) def intersect_instance_callable(self, typ: Instance, callable_type: CallableType) -> Instance: """Creates a fake type that represents the intersection of an Instance and a CallableType. @@ -4656,7 +4911,7 @@ def intersect_instance_callable(self, typ: Instance, callable_type: CallableType cur_module.names[gen_name] = SymbolTableNode(GDEF, info) - return Instance(info, []) + return Instance(info, [], extra_attrs=typ.extra_attrs) def make_fake_callable(self, typ: Instance) -> Instance: """Produce a new type that makes type Callable with a generic callable type.""" @@ -4960,6 +5215,12 @@ def find_isinstance_check_helper(self, node: Expression) -> tuple[TypeMap, TypeM if literal(expr) == LITERAL_TYPE: vartype = self.lookup_type(expr) return self.conditional_callable_type_map(expr, vartype) + elif refers_to_fullname(node.callee, "builtins.hasattr"): + if len(node.args) != 2: # the error will be reported elsewhere + return {}, {} + attr = try_getting_str_literals(node.args[1], self.lookup_type(node.args[1])) + if literal(expr) == LITERAL_TYPE and attr and len(attr) == 1: + return self.hasattr_type_maps(expr, self.lookup_type(expr), attr[0]) elif isinstance(node.callee, RefExpr): if node.callee.type_guard is not None: # TODO: Follow keyword args or *args, **kwargs @@ -5243,7 +5504,7 @@ def refine_parent_types(self, expr: Expression, expr_type: Type) -> Mapping[Expr # and create function that will try replaying the same lookup # operation against arbitrary types. if isinstance(expr, MemberExpr): - parent_expr = expr.expr + parent_expr = collapse_walrus(expr.expr) parent_type = self.lookup_type_or_none(parent_expr) member_name = expr.name @@ -5267,7 +5528,7 @@ def replay_lookup(new_parent_type: ProperType) -> Type | None: return member_type elif isinstance(expr, IndexExpr): - parent_expr = expr.base + parent_expr = collapse_walrus(expr.base) parent_type = self.lookup_type_or_none(parent_expr) index_type = self.lookup_type_or_none(expr.index) @@ -5509,16 +5770,47 @@ def refine_away_none_in_comparison( # # Helpers # + @overload + def check_subtype( + self, + subtype: Type, + supertype: Type, + context: Context, + msg: str, + subtype_label: str | None = None, + supertype_label: str | None = None, + *, + notes: list[str] | None = None, + code: ErrorCode | None = None, + outer_context: Context | None = None, + ) -> bool: + ... + @overload def check_subtype( self, subtype: Type, supertype: Type, context: Context, - msg: str | ErrorMessage = message_registry.INCOMPATIBLE_TYPES, + msg: ErrorMessage, subtype_label: str | None = None, supertype_label: str | None = None, *, + notes: list[str] | None = None, + outer_context: Context | None = None, + ) -> bool: + ... + + def check_subtype( + self, + subtype: Type, + supertype: Type, + context: Context, + msg: str | ErrorMessage, + subtype_label: str | None = None, + supertype_label: str | None = None, + *, + notes: list[str] | None = None, code: ErrorCode | None = None, outer_context: Context | None = None, ) -> bool: @@ -5526,24 +5818,20 @@ def check_subtype( if is_subtype(subtype, supertype, options=self.options): return True - if isinstance(msg, ErrorMessage): - msg_text = msg.value - code = msg.code - else: - msg_text = msg + if isinstance(msg, str): + msg = ErrorMessage(msg, code=code) + orig_subtype = subtype subtype = get_proper_type(subtype) orig_supertype = supertype supertype = get_proper_type(supertype) if self.msg.try_report_long_tuple_assignment_error( - subtype, supertype, context, msg_text, subtype_label, supertype_label, code=code + subtype, supertype, context, msg, subtype_label, supertype_label ): return False - if self.should_suppress_optional_error([subtype]): - return False extra_info: list[str] = [] note_msg = "" - notes: list[str] = [] + notes = notes or [] if subtype_label is not None or supertype_label is not None: subtype_str, supertype_str = format_type_distinctly(orig_subtype, orig_supertype) if subtype_label is not None: @@ -5554,31 +5842,31 @@ def check_subtype( outer_context or context, subtype, supertype, supertype_str ) if isinstance(subtype, Instance) and isinstance(supertype, Instance): - notes = append_invariance_notes([], subtype, supertype) + notes = append_invariance_notes(notes, subtype, supertype) if extra_info: - msg_text += " (" + ", ".join(extra_info) + ")" + msg = msg.with_additional_msg(" (" + ", ".join(extra_info) + ")") - self.fail(ErrorMessage(msg_text, code=code), context) + self.fail(msg, context) for note in notes: - self.msg.note(note, context, code=code) + self.msg.note(note, context, code=msg.code) if note_msg: - self.note(note_msg, context, code=code) - self.msg.maybe_note_concatenate_pos_args(subtype, supertype, context, code=code) + self.note(note_msg, context, code=msg.code) + self.msg.maybe_note_concatenate_pos_args(subtype, supertype, context, code=msg.code) if ( isinstance(supertype, Instance) and supertype.type.is_protocol and isinstance(subtype, (Instance, TupleType, TypedDictType)) ): - self.msg.report_protocol_problems(subtype, supertype, context, code=code) + self.msg.report_protocol_problems(subtype, supertype, context, code=msg.code) if isinstance(supertype, CallableType) and isinstance(subtype, Instance): call = find_member("__call__", subtype, subtype, is_operator=True) if call: - self.msg.note_call(subtype, call, context, code=code) + self.msg.note_call(subtype, call, context, code=msg.code) if isinstance(subtype, (CallableType, Overloaded)) and isinstance(supertype, Instance): if supertype.type.is_protocol and supertype.type.protocol_members == ["__call__"]: call = find_member("__call__", supertype, subtype, is_operator=True) assert call is not None - self.msg.note_call(supertype, call, context, code=code) + self.msg.note_call(supertype, call, context, code=msg.code) self.check_possible_missing_await(subtype, supertype, context) return False @@ -5623,7 +5911,9 @@ def check_possible_missing_await( aw_type = self.get_precise_awaitable_type(subtype, local_errors) if aw_type is None: return - if not self.check_subtype(aw_type, supertype, context): + if not self.check_subtype( + aw_type, supertype, context, msg=message_registry.INCOMPATIBLE_TYPES + ): return self.msg.possible_missing_await(context) @@ -5640,9 +5930,6 @@ def contains_none(self, t: Type) -> bool: ) ) - def should_suppress_optional_error(self, related_types: list[Type]) -> bool: - return self.suppress_none_errors and any(self.contains_none(t) for t in related_types) - def named_type(self, name: str) -> Instance: """Return an instance type with given name and implicit Any type args. @@ -5652,7 +5939,7 @@ def named_type(self, name: str) -> Instance: sym = self.lookup_qualified(name) node = sym.node if isinstance(node, TypeAlias): - assert isinstance(node.target, Instance) # type: ignore + assert isinstance(node.target, Instance) # type: ignore[misc] node = node.target.type assert isinstance(node, TypeInfo) any_type = AnyType(TypeOfAny.from_omitted_generics) @@ -5829,7 +6116,9 @@ def enter_partial_types( self.msg.need_annotation_for_var(var, context, self.options.python_version) self.partial_reported.add(var) if var.type: - var.type = self.fixup_partial_type(var.type) + fixed = self.fixup_partial_type(var.type) + var.invalid_partial_type = fixed != var.type + var.type = fixed def handle_partial_var_type( self, typ: PartialType, is_lvalue: bool, node: Var, context: Context @@ -5933,9 +6222,17 @@ def fail( self.msg.fail(msg, context, code=code) def note( - self, msg: str, context: Context, offset: int = 0, *, code: ErrorCode | None = None + self, + msg: str | ErrorMessage, + context: Context, + offset: int = 0, + *, + code: ErrorCode | None = None, ) -> None: """Produce a note.""" + if isinstance(msg, ErrorMessage): + self.msg.note(msg.value, context, code=msg.code) + return self.msg.note(msg, context, offset=offset, code=code) def iterable_item_type(self, instance: Instance) -> Type: @@ -5985,7 +6282,7 @@ def infer_issubclass_maps(self, node: CallExpr, expr: Expression) -> tuple[TypeM vartype = UnionType(union_list) elif isinstance(vartype, TypeType): vartype = vartype.item - elif isinstance(vartype, Instance) and vartype.type.fullname == "builtins.type": + elif isinstance(vartype, Instance) and vartype.type.is_metaclass(): vartype = self.named_type("builtins.object") else: # Any other object whose type we don't know precisely @@ -6062,6 +6359,8 @@ def conditional_types_with_intersection( def is_writable_attribute(self, node: Node) -> bool: """Check if an attribute is writable""" if isinstance(node, Var): + if node.is_property and not node.is_settable_property: + return False return True elif isinstance(node, OverloadedFuncDef) and node.is_property: first_item = cast(Decorator, node.items[0]) @@ -6140,8 +6439,104 @@ class Foo(Enum): and member_type.fallback.type == parent_type.type_object() ) + def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: + """Inject an extra attribute with Any type using fallbacks.""" + orig_typ = typ + typ = get_proper_type(typ) + any_type = AnyType(TypeOfAny.unannotated) + if isinstance(typ, Instance): + result = typ.copy_with_extra_attr(name, any_type) + # For instances, we erase the possible module name, so that restrictions + # become anonymous types.ModuleType instances, allowing hasattr() to + # have effect on modules. + assert result.extra_attrs is not None + result.extra_attrs.mod_name = None + return result + if isinstance(typ, TupleType): + fallback = typ.partial_fallback.copy_with_extra_attr(name, any_type) + return typ.copy_modified(fallback=fallback) + if isinstance(typ, CallableType): + fallback = typ.fallback.copy_with_extra_attr(name, any_type) + return typ.copy_modified(fallback=fallback) + if isinstance(typ, TypeType) and isinstance(typ.item, Instance): + return TypeType.make_normalized(self.add_any_attribute_to_type(typ.item, name)) + if isinstance(typ, TypeVarType): + return typ.copy_modified( + upper_bound=self.add_any_attribute_to_type(typ.upper_bound, name), + values=[self.add_any_attribute_to_type(v, name) for v in typ.values], + ) + if isinstance(typ, UnionType): + with_attr, without_attr = self.partition_union_by_attr(typ, name) + return make_simplified_union( + with_attr + [self.add_any_attribute_to_type(typ, name) for typ in without_attr] + ) + return orig_typ + + def hasattr_type_maps( + self, expr: Expression, source_type: Type, name: str + ) -> tuple[TypeMap, TypeMap]: + """Simple support for hasattr() checks. -class CollectArgTypes(TypeTraverserVisitor): + Essentially the logic is following: + * In the if branch, keep types that already has a valid attribute as is, + for other inject an attribute with `Any` type. + * In the else branch, remove types that already have a valid attribute, + while keeping the rest. + """ + if self.has_valid_attribute(source_type, name): + return {expr: source_type}, {} + + source_type = get_proper_type(source_type) + if isinstance(source_type, UnionType): + _, without_attr = self.partition_union_by_attr(source_type, name) + yes_map = {expr: self.add_any_attribute_to_type(source_type, name)} + return yes_map, {expr: make_simplified_union(without_attr)} + + type_with_attr = self.add_any_attribute_to_type(source_type, name) + if type_with_attr != source_type: + return {expr: type_with_attr}, {} + return {}, {} + + def partition_union_by_attr( + self, source_type: UnionType, name: str + ) -> tuple[list[Type], list[Type]]: + with_attr = [] + without_attr = [] + for item in source_type.items: + if self.has_valid_attribute(item, name): + with_attr.append(item) + else: + without_attr.append(item) + return with_attr, without_attr + + def has_valid_attribute(self, typ: Type, name: str) -> bool: + p_typ = get_proper_type(typ) + if isinstance(p_typ, AnyType): + return False + if isinstance(p_typ, Instance) and p_typ.extra_attrs and p_typ.extra_attrs.mod_name: + # Presence of module_symbol_table means this check will skip ModuleType.__getattr__ + module_symbol_table = p_typ.type.names + else: + module_symbol_table = None + with self.msg.filter_errors() as watcher: + analyze_member_access( + name, + typ, + TempNode(AnyType(TypeOfAny.special_form)), + False, + False, + False, + self.msg, + original_type=typ, + chk=self, + # This is not a real attribute lookup so don't mess with deferring nodes. + no_deferral=True, + module_symbol_table=module_symbol_table, + ) + return not watcher.has_new_errors() + + +class CollectArgTypeVarTypes(TypeTraverserVisitor): """Collects the non-nested argument types in a set.""" def __init__(self) -> None: @@ -6193,11 +6588,11 @@ def conditional_types( return proposed_type, default elif not any( type_range.is_upper_bound for type_range in proposed_type_ranges - ) and is_proper_subtype(current_type, proposed_type): + ) and is_proper_subtype(current_type, proposed_type, ignore_promotions=True): # Expression is always of one of the types in proposed_type_ranges return default, UninhabitedType() elif not is_overlapping_types( - current_type, proposed_type, prohibit_none_typevar_overlap=True + current_type, proposed_type, prohibit_none_typevar_overlap=True, ignore_promotions=True ): # Expression is never of any type in proposed_type_ranges return UninhabitedType(), default @@ -6481,7 +6876,7 @@ def is_unsafe_overlapping_overload_signatures( return is_callable_compatible( signature, other, - is_compat=is_overlapping_types_no_promote, + is_compat=is_overlapping_types_no_promote_no_uninhabited, is_compat_return=lambda l, r: not is_subtype_no_promote(l, r), ignore_return=False, check_args_covariantly=True, @@ -6489,7 +6884,7 @@ def is_unsafe_overlapping_overload_signatures( ) or is_callable_compatible( other, signature, - is_compat=is_overlapping_types_no_promote, + is_compat=is_overlapping_types_no_promote_no_uninhabited, is_compat_return=lambda l, r: not is_subtype_no_promote(r, l), ignore_return=False, check_args_covariantly=False, @@ -6562,7 +6957,6 @@ def overload_can_never_match(signature: CallableType, other: CallableType) -> bo exp_signature = expand_type( signature, {tvar.id: erase_def_to_union_or_bound(tvar) for tvar in signature.variables} ) - assert isinstance(exp_signature, ProperType) assert isinstance(exp_signature, CallableType) return is_callable_compatible( exp_signature, other, is_compat=is_more_precise, ignore_return=True @@ -6969,12 +7363,33 @@ def is_static(func: FuncBase | Decorator) -> bool: assert False, f"Unexpected func type: {type(func)}" +def is_property(defn: SymbolNode) -> bool: + if isinstance(defn, Decorator): + return defn.func.is_property + if isinstance(defn, OverloadedFuncDef): + if defn.items and isinstance(defn.items[0], Decorator): + return defn.items[0].func.is_property + return False + + +def get_property_type(t: ProperType) -> ProperType: + if isinstance(t, CallableType): + return get_proper_type(t.ret_type) + if isinstance(t, Overloaded): + return get_proper_type(t.items[0].ret_type) + return t + + def is_subtype_no_promote(left: Type, right: Type) -> bool: return is_subtype(left, right, ignore_promotions=True) -def is_overlapping_types_no_promote(left: Type, right: Type) -> bool: - return is_overlapping_types(left, right, ignore_promotions=True) +def is_overlapping_types_no_promote_no_uninhabited(left: Type, right: Type) -> bool: + # For the purpose of unsafe overload checks we consider list[] and list[int] + # non-overlapping. This is consistent with how we treat list[int] and list[str] as + # non-overlapping, despite [] belongs to both. Also this will prevent false positives + # for failed type inference during unification. + return is_overlapping_types(left, right, ignore_promotions=True, ignore_uninhabited=True) def is_private(node_name: str) -> bool: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index cb542ee5300b..ac16f9c9c813 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -127,6 +127,7 @@ CallableType, DeletedType, ErasedType, + ExtraAttrs, FunctionLike, Instance, LiteralType, @@ -140,9 +141,11 @@ StarType, TupleType, Type, + TypeAliasType, TypedDictType, TypeOfAny, TypeType, + TypeVarTupleType, TypeVarType, UninhabitedType, UnionType, @@ -153,6 +156,7 @@ is_generic_instance, is_named_instance, is_optional, + is_self_type_like, remove_optional, ) from mypy.typestate import TypeState @@ -193,10 +197,12 @@ class TooManyUnions(Exception): """ -def allow_fast_container_literal(t: ProperType) -> bool: +def allow_fast_container_literal(t: Type) -> bool: + if isinstance(t, TypeAliasType) and t.is_recursive: + return False + t = get_proper_type(t) return isinstance(t, Instance) or ( - isinstance(t, TupleType) - and all(allow_fast_container_literal(get_proper_type(it)) for it in t.items) + isinstance(t, TupleType) and all(allow_fast_container_literal(it) for it in t.items) ) @@ -318,7 +324,7 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: result = self.typeddict_callable(node) else: result = type_object_type(node, self.named_type) - if isinstance(result, CallableType) and isinstance( # type: ignore + if isinstance(result, CallableType) and isinstance( # type: ignore[misc] result.ret_type, Instance ): # We need to set correct line and column @@ -331,13 +337,7 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: result = erasetype.erase_typevars(result) elif isinstance(node, MypyFile): # Reference to a module object. - try: - result = self.named_type("types.ModuleType") - except KeyError: - # In test cases might 'types' may not be available. - # Fall back to a dummy 'object' type instead to - # avoid a crash. - result = self.named_type("builtins.object") + result = self.module_type(node) elif isinstance(node, Decorator): result = self.analyze_var_ref(node.var, e) elif isinstance(node, TypeAlias): @@ -345,7 +345,7 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: # Note that we suppress bogus errors for alias redefinitions, # they are already reported in semanal.py. result = self.alias_type_in_runtime_context( - node, node.no_args, e, alias_definition=e.is_alias_rvalue or lvalue + node, ctx=e, alias_definition=e.is_alias_rvalue or lvalue ) elif isinstance(node, (TypeVarExpr, ParamSpecExpr)): result = self.object_type() @@ -373,6 +373,31 @@ def analyze_var_ref(self, var: Var, context: Context) -> Type: # Implicit 'Any' type. return AnyType(TypeOfAny.special_form) + def module_type(self, node: MypyFile) -> Instance: + try: + result = self.named_type("types.ModuleType") + except KeyError: + # In test cases might 'types' may not be available. + # Fall back to a dummy 'object' type instead to + # avoid a crash. + result = self.named_type("builtins.object") + module_attrs = {} + immutable = set() + for name, n in node.names.items(): + if not n.module_public: + continue + if isinstance(n.node, Var) and n.node.is_final: + immutable.add(name) + typ = self.chk.determine_type_of_member(n) + if typ: + module_attrs[name] = typ + else: + # TODO: what to do about nested module references? + # They are non-trivial because there may be import cycles. + module_attrs[name] = AnyType(TypeOfAny.special_form) + result.extra_attrs = ExtraAttrs(module_attrs, immutable, node.fullname) + return result + def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: """Type check a call expression.""" if e.analyzed: @@ -800,10 +825,11 @@ def check_typeddict_call_with_kwargs( lvalue_type=item_expected_type, rvalue=item_value, context=item_value, - msg=message_registry.INCOMPATIBLE_TYPES, + msg=ErrorMessage( + message_registry.INCOMPATIBLE_TYPES.value, code=codes.TYPEDDICT_ITEM + ), lvalue_name=f'TypedDict item "{item_name}"', rvalue_name="expression", - code=codes.TYPEDDICT_ITEM, ) return orig_ret_type @@ -1173,7 +1199,7 @@ def check_call_expr_with_callee_type( return ret_type def check_union_call_expr(self, e: CallExpr, object_type: UnionType, member: str) -> Type: - """ "Type check calling a member expression where the base type is a union.""" + """Type check calling a member expression where the base type is a union.""" res: list[Type] = [] for typ in object_type.relevant_items(): # Member access errors are already reported when visiting the member expression. @@ -1322,6 +1348,8 @@ def check_callable_call( See the docstring of check_call for more information. """ + # Always unpack **kwargs before checking a call. + callee = callee.with_unpacked_kwargs() if callable_name is None and callee.name: callable_name = callee.name ret_type = get_proper_type(callee.ret_type) @@ -1370,7 +1398,9 @@ def check_callable_call( ) if callee.is_generic(): - need_refresh = any(isinstance(v, ParamSpecType) for v in callee.variables) + need_refresh = any( + isinstance(v, (ParamSpecType, TypeVarTupleType)) for v in callee.variables + ) callee = freshen_function_type_vars(callee) callee = self.infer_function_type_arguments_using_context(callee, context) callee = self.infer_function_type_arguments( @@ -1482,7 +1512,7 @@ def analyze_type_type_callee(self, item: ProperType, context: Context) -> Type: res = type_object_type(item.type, self.named_type) if isinstance(res, CallableType): res = res.copy_modified(from_type_type=True) - expanded = get_proper_type(expand_type_by_instance(res, item)) + expanded = expand_type_by_instance(res, item) if isinstance(expanded, CallableType): # Callee of the form Type[...] should never be generic, only # proper class objects can be. @@ -2029,8 +2059,6 @@ def check_arg( ): self.msg.concrete_only_call(callee_type, context) elif not is_subtype(caller_type, callee_type, options=self.chk.options): - if self.chk.should_suppress_optional_error([caller_type, callee_type]): - return code = self.msg.incompatible_argument( n, m, @@ -2057,6 +2085,8 @@ def check_overload_call( context: Context, ) -> tuple[Type, Type]: """Checks a call to an overloaded function.""" + # Normalize unpacked kwargs before checking the call. + callee = callee.with_unpacked_kwargs() arg_types = self.infer_arg_types_in_empty_context(args) # Step 1: Filter call targets to remove ones where the argument counts don't match plausible_targets = self.plausible_overload_call_targets( @@ -2150,13 +2180,11 @@ def check_overload_call( else: # There was no plausible match: give up target = AnyType(TypeOfAny.from_error) - - if not self.chk.should_suppress_optional_error(arg_types): - if not is_operator_method(callable_name): - code = None - else: - code = codes.OPERATOR - self.msg.no_variant_matches_arguments(callee, arg_types, context, code=code) + if not is_operator_method(callable_name): + code = None + else: + code = codes.OPERATOR + self.msg.no_variant_matches_arguments(callee, arg_types, context, code=code) result = self.check_call( target, @@ -3370,7 +3398,8 @@ def check_boolean_op(self, e: OpExpr, context: Context) -> Type: assert e.op in ("and", "or") # Checked by visit_op_expr if e.right_always: - left_map, right_map = None, {} # type: mypy.checker.TypeMap, mypy.checker.TypeMap + left_map: mypy.checker.TypeMap = None + right_map: mypy.checker.TypeMap = {} elif e.right_unreachable: left_map, right_map = {}, None elif e.op == "and": @@ -3701,6 +3730,11 @@ def visit_assert_type_expr(self, expr: AssertTypeExpr) -> Type: ) target_type = expr.type if not is_same_type(source_type, target_type): + if not self.chk.in_checked_function(): + self.msg.note( + '"assert_type" expects everything to be "Any" in unchecked functions', + expr.expr, + ) self.msg.assert_type_fail(source_type, target_type, expr) return source_type @@ -3783,12 +3817,10 @@ def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type: both `reveal_type` instances will reveal the same type `def (...) -> builtins.list[Any]`. Note that type variables are implicitly substituted with `Any`. """ - return self.alias_type_in_runtime_context( - alias.node, alias.no_args, alias, alias_definition=True - ) + return self.alias_type_in_runtime_context(alias.node, ctx=alias, alias_definition=True) def alias_type_in_runtime_context( - self, alias: TypeAlias, no_args: bool, ctx: Context, *, alias_definition: bool = False + self, alias: TypeAlias, *, ctx: Context, alias_definition: bool = False ) -> Type: """Get type of a type alias (could be generic) in a runtime expression. @@ -3803,7 +3835,7 @@ class LongName(Generic[T]): ... x = A() y = cast(A, ...) """ - if isinstance(alias.target, Instance) and alias.target.invalid: # type: ignore + if isinstance(alias.target, Instance) and alias.target.invalid: # type: ignore[misc] # An invalid alias, error already has been reported return AnyType(TypeOfAny.from_error) # If this is a generic alias, we set all variables to `Any`. @@ -3820,7 +3852,7 @@ class LongName(Generic[T]): ... # Normally we get a callable type (or overloaded) with .is_type_obj() true # representing the class's constructor tp = type_object_type(item.type, self.named_type) - if no_args: + if alias.no_args: return tp return self.apply_type_arguments_to_callable(tp, item.args, ctx) elif ( @@ -3838,6 +3870,7 @@ class LongName(Generic[T]): ... if alias_definition: return AnyType(TypeOfAny.special_form) # This type is invalid in most runtime contexts, give it an 'object' type. + # TODO: Use typing._SpecialForm instead? return self.named_type("builtins.object") def apply_type_arguments_to_callable( @@ -4208,6 +4241,10 @@ def infer_lambda_type_using_context( callable_ctx = get_proper_type(replace_meta_vars(ctx, ErasedType())) assert isinstance(callable_ctx, CallableType) + # The callable_ctx may have a fallback of builtins.type if the context + # is a constructor -- but this fallback doesn't make sense for lambdas. + callable_ctx = callable_ctx.copy_modified(fallback=self.named_type("builtins.function")) + if callable_ctx.type_guard is not None: # Lambda's return type cannot be treated as a `TypeGuard`, # because it is implicit. And `TypeGuard`s must be explicit. @@ -4263,9 +4300,22 @@ def visit_super_expr(self, e: SuperExpr) -> Type: # The base is the first MRO entry *after* type_info that has a member # with the right name - try: + index = None + if type_info in mro: index = mro.index(type_info) - except ValueError: + else: + method = self.chk.scope.top_function() + assert method is not None + # Mypy explicitly allows supertype upper bounds (and no upper bound at all) + # for annotating self-types. However, if such an annotation is used for + # checking super() we will still get an error. So to be consistent, we also + # allow such imprecise annotations for use with super(), where we fall back + # to the current class MRO instead. + if is_self_type_like(instance_type, is_classmethod=method.is_class): + if e.info and type_info in e.info.mro: + mro = e.info.mro + index = mro.index(type_info) + if index is None: self.chk.fail(message_registry.SUPER_ARG_2_NOT_INSTANCE_OF_ARG_1, e) return AnyType(TypeOfAny.from_error) @@ -4559,7 +4609,7 @@ def visit_conditional_expr(self, e: ConditionalExpr, allow_none_return: bool = F # # TODO: Always create a union or at least in more cases? if isinstance(get_proper_type(self.type_context[-1]), UnionType): - res = make_simplified_union([if_type, full_context_else_type]) + res: Type = make_simplified_union([if_type, full_context_else_type]) else: res = join.join_types(if_type, else_type) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 6777c4354a04..6c9da4a6ce7c 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -31,7 +31,6 @@ is_final_node, ) from mypy.plugin import AttributeContext -from mypy.typeanal import set_any_tvars from mypy.typeops import ( bind_self, class_callable, @@ -90,6 +89,7 @@ def __init__( chk: mypy.checker.TypeChecker, self_type: Type | None, module_symbol_table: SymbolTable | None = None, + no_deferral: bool = False, ) -> None: self.is_lvalue = is_lvalue self.is_super = is_super @@ -100,6 +100,7 @@ def __init__( self.msg = msg self.chk = chk self.module_symbol_table = module_symbol_table + self.no_deferral = no_deferral def named_type(self, name: str) -> Instance: return self.chk.named_type(name) @@ -124,6 +125,7 @@ def copy_modified( self.chk, self.self_type, self.module_symbol_table, + self.no_deferral, ) if messages is not None: mx.msg = messages @@ -149,6 +151,7 @@ def analyze_member_access( in_literal_context: bool = False, self_type: Type | None = None, module_symbol_table: SymbolTable | None = None, + no_deferral: bool = False, ) -> Type: """Return the type of attribute 'name' of 'typ'. @@ -183,6 +186,7 @@ def analyze_member_access( chk=chk, self_type=self_type, module_symbol_table=module_symbol_table, + no_deferral=no_deferral, ) result = _analyze_member_access(name, typ, mx, override_info) possible_literal = get_proper_type(result) @@ -232,8 +236,6 @@ def _analyze_member_access( elif isinstance(typ, DeletedType): mx.msg.deleted_as_rvalue(typ, mx.context) return AnyType(TypeOfAny.from_error) - if mx.chk.should_suppress_optional_error([typ]): - return AnyType(TypeOfAny.from_error) return report_missing_attribute(mx.original_type, typ, name, mx) @@ -294,6 +296,9 @@ def analyze_instance_member_access( # Look up the member. First look up the method dictionary. method = info.get_method(name) if method and not isinstance(method, Decorator): + if mx.is_super: + validate_super_call(method, mx) + if method.is_property: assert isinstance(method, OverloadedFuncDef) first_item = cast(Decorator, method.items[0]) @@ -302,12 +307,12 @@ def analyze_instance_member_access( mx.msg.cant_assign_to_method(mx.context) signature = function_type(method, mx.named_type("builtins.function")) signature = freshen_function_type_vars(signature) - if name == "__new__": + if name == "__new__" or method.is_static: # __new__ is special and behaves like a static method -- don't strip # the first argument. pass else: - if isinstance(signature, FunctionLike) and name != "__call__": + if name != "__call__": # TODO: use proper treatment of special methods on unions instead # of this hack here and below (i.e. mx.self_type). dispatched_type = meet.meet_types(mx.original_type, typ) @@ -315,6 +320,8 @@ def analyze_instance_member_access( signature, dispatched_type, method.is_class, mx.context, name, mx.msg ) signature = bind_self(signature, mx.self_type, is_classmethod=method.is_class) + # TODO: should we skip these steps for static methods as well? + # Since generic static methods should not be allowed. typ = map_instance_to_supertype(typ, method.info) member_type = expand_type_by_instance(signature, typ) freeze_type_vars(member_type) @@ -324,6 +331,25 @@ def analyze_instance_member_access( return analyze_member_var_access(name, typ, info, mx) +def validate_super_call(node: FuncBase, mx: MemberContext) -> None: + unsafe_super = False + if isinstance(node, FuncDef) and node.is_trivial_body: + unsafe_super = True + impl = node + elif isinstance(node, OverloadedFuncDef): + if node.impl: + impl = node.impl if isinstance(node.impl, FuncDef) else node.impl.func + unsafe_super = impl.is_trivial_body + if unsafe_super: + ret_type = ( + impl.type.ret_type + if isinstance(impl.type, CallableType) + else AnyType(TypeOfAny.unannotated) + ) + if not subtypes.is_subtype(NoneType(), ret_type): + mx.msg.unsafe_super(node.name, node.info.name, mx.context) + + def analyze_type_callable_member_access(name: str, typ: FunctionLike, mx: MemberContext) -> Type: # Class attribute. # TODO super? @@ -425,8 +451,6 @@ def analyze_none_member_access(name: str, typ: NoneType, mx: MemberContext) -> T ret_type=literal_false, fallback=mx.named_type("builtins.function"), ) - elif mx.chk.should_suppress_optional_error([typ]): - return AnyType(TypeOfAny.from_error) else: return _analyze_member_access(name, mx.named_type("builtins.object"), mx) @@ -447,6 +471,8 @@ def analyze_member_var_access( if isinstance(vv, Decorator): # The associated Var node of a decorator contains the type. v = vv.var + if mx.is_super: + validate_super_call(vv.func, mx) if isinstance(vv, TypeInfo): # If the associated variable is a TypeInfo synthesize a Var node for @@ -455,14 +481,16 @@ def analyze_member_var_access( v = Var(name, type=type_object_type(vv, mx.named_type)) v.info = info - if isinstance(vv, TypeAlias) and isinstance(get_proper_type(vv.target), Instance): + if isinstance(vv, TypeAlias): # Similar to the above TypeInfo case, we allow using # qualified type aliases in runtime context if it refers to an # instance type. For example: # class C: # A = List[int] # x = C.A() <- this is OK - typ = instance_alias_type(vv, mx.named_type) + typ = mx.chk.expr_checker.alias_type_in_runtime_context( + vv, ctx=mx.context, alias_definition=mx.is_lvalue + ) v = Var(name, type=typ) v.info = info @@ -477,6 +505,9 @@ def analyze_member_var_access( return analyze_var(name, v, itype, info, mx, implicit=implicit) elif isinstance(v, FuncDef): assert False, "Did not expect a function" + elif isinstance(v, MypyFile): + mx.chk.module_refs.add(v.fullname) + return mx.chk.expr_checker.module_type(v) elif ( not v and name not in ["__getattr__", "__setattr__", "__getattribute__"] @@ -539,12 +570,15 @@ def analyze_member_var_access( return AnyType(TypeOfAny.special_form) # Could not find the member. + if itype.extra_attrs and name in itype.extra_attrs.attrs: + # For modules use direct symbol table lookup. + if not itype.extra_attrs.mod_name: + return itype.extra_attrs.attrs[name] + if mx.is_super: mx.msg.undefined_in_superclass(name, mx.context) return AnyType(TypeOfAny.from_error) else: - if mx.chk and mx.chk.should_suppress_optional_error([itype]): - return AnyType(TypeOfAny.from_error) return report_missing_attribute(mx.original_type, itype, name, mx) @@ -593,7 +627,7 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: itype=descriptor_type, info=descriptor_type.type, self_type=descriptor_type, - name="__set__", + name="__get__", mx=mx, ) @@ -648,21 +682,6 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: return inferred_dunder_get_type.ret_type -def instance_alias_type(alias: TypeAlias, named_type: Callable[[str], Instance]) -> Type: - """Type of a type alias node targeting an instance, when appears in runtime context. - - As usual, we first erase any unbound type variables to Any. - """ - target: Type = get_proper_type(alias.target) - assert isinstance( - get_proper_type(target), Instance - ), "Must be called only with aliases to classes" - target = get_proper_type(set_any_tvars(alias, alias.line, alias.column)) - assert isinstance(target, Instance) - tp = type_object_type(target.type, named_type) - return expand_type_by_instance(tp, target) - - def is_instance_var(var: Var, info: TypeInfo) -> bool: """Return if var is an instance variable according to PEP 526.""" return ( @@ -736,7 +755,7 @@ def analyze_var( signature, dispatched_type, var.is_classmethod, mx.context, name, mx.msg ) signature = bind_self(signature, mx.self_type, var.is_classmethod) - expanded_signature = get_proper_type(expand_type_by_instance(signature, itype)) + expanded_signature = expand_type_by_instance(signature, itype) freeze_type_vars(expanded_signature) if var.is_property: # A property cannot have an overloaded type => the cast is fine. @@ -745,7 +764,7 @@ def analyze_var( else: result = expanded_signature else: - if not var.is_ready: + if not var.is_ready and not mx.no_deferral: mx.not_ready_callback(var.name, mx.context) # Implicit 'Any' type. result = AnyType(TypeOfAny.special_form) @@ -859,6 +878,10 @@ def analyze_class_attribute_access( node = info.get(name) if not node: + if itype.extra_attrs and name in itype.extra_attrs.attrs: + # For modules use direct symbol table lookup. + if not itype.extra_attrs.mod_name: + return itype.extra_attrs.attrs[name] if info.fallback_to_any: return apply_class_attr_hook(mx, hook, AnyType(TypeOfAny.special_form)) return None @@ -967,10 +990,10 @@ def analyze_class_attribute_access( # Reference to a module object. return mx.named_type("types.ModuleType") - if isinstance(node.node, TypeAlias) and isinstance( - get_proper_type(node.node.target), Instance - ): - return instance_alias_type(node.node, mx.named_type) + if isinstance(node.node, TypeAlias): + return mx.chk.expr_checker.alias_type_in_runtime_context( + node.node, ctx=mx.context, alias_definition=mx.is_lvalue + ) if is_decorated: assert isinstance(node.node, Decorator) @@ -1112,7 +1135,7 @@ class B(A[str]): pass ] ) if isuper is not None: - t = cast(ProperType, expand_type_by_instance(t, isuper)) + t = expand_type_by_instance(t, isuper) return t diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index b8720d9402f8..603b392eee29 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -33,6 +33,7 @@ coerce_to_literal, make_simplified_union, try_getting_str_literals_from_type, + tuple_fallback, ) from mypy.types import ( AnyType, @@ -256,16 +257,13 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType: contracted_inner_types = self.contract_starred_pattern_types( inner_types, star_position, required_patterns ) - can_match = True for p, t in zip(o.patterns, contracted_inner_types): pattern_type = self.accept(p, t) typ, rest, type_map = pattern_type - if is_uninhabited(typ): - can_match = False - else: - contracted_new_inner_types.append(typ) - contracted_rest_inner_types.append(rest) + contracted_new_inner_types.append(typ) + contracted_rest_inner_types.append(rest) self.update_type_map(captures, type_map) + new_inner_types = self.expand_starred_pattern_types( contracted_new_inner_types, star_position, len(inner_types) ) @@ -278,9 +276,7 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType: # new_type: Type rest_type: Type = current_type - if not can_match: - new_type = UninhabitedType() - elif isinstance(current_type, TupleType): + if isinstance(current_type, TupleType): narrowed_inner_types = [] inner_rest_types = [] for inner_type, new_inner_type in zip(inner_types, new_inner_types): @@ -325,7 +321,9 @@ def get_sequence_type(self, t: Type) -> Type | None: else: return None - if self.chk.type_is_iterable(t) and isinstance(t, Instance): + if self.chk.type_is_iterable(t) and isinstance(t, (Instance, TupleType)): + if isinstance(t, TupleType): + t = tuple_fallback(t) return self.chk.iterable_item_type(t) else: return None @@ -645,6 +643,9 @@ def construct_sequence_child(self, outer_type: Type, inner_type: Type) -> Type: For example: construct_sequence_child(List[int], str) = List[str] + + TODO: this doesn't make sense. For example if one has class S(Sequence[int], Generic[T]) + or class T(Sequence[Tuple[T, T]]), there is no way any of those can map to Sequence[str]. """ proper_type = get_proper_type(outer_type) if isinstance(proper_type, UnionType): @@ -657,6 +658,8 @@ def construct_sequence_child(self, outer_type: Type, inner_type: Type) -> Type: sequence = self.chk.named_generic_type("typing.Sequence", [inner_type]) if is_subtype(outer_type, self.chk.named_type("typing.Sequence")): proper_type = get_proper_type(outer_type) + if isinstance(proper_type, TupleType): + proper_type = tuple_fallback(proper_type) assert isinstance(proper_type, Instance) empty_type = fill_typevars(proper_type.type) partial_type = expand_type_by_instance(empty_type, sequence) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 55cc0fea3720..485d2f67f5de 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -8,6 +8,8 @@ import sys from io import StringIO +from mypy.errorcodes import error_codes + if sys.version_info >= (3, 11): import tomllib else: @@ -69,6 +71,15 @@ def try_split(v: str | Sequence[str], split_regex: str = "[,]") -> list[str]: return [p.strip() for p in v] +def validate_codes(codes: list[str]) -> list[str]: + invalid_codes = set(codes) - set(error_codes.keys()) + if invalid_codes: + raise argparse.ArgumentTypeError( + f"Invalid error code(s): {', '.join(sorted(invalid_codes))}" + ) + return codes + + def expand_path(path: str) -> str: """Expand the user home directory and any environment variables contained within the provided path. @@ -132,28 +143,27 @@ def check_follow_imports(choice: str) -> str: # types. ini_config_types: Final[dict[str, _INI_PARSER_CALLABLE]] = { "python_version": parse_version, - "strict_optional_whitelist": lambda s: s.split(), "custom_typing_module": str, "custom_typeshed_dir": expand_path, "mypy_path": lambda s: [expand_path(p.strip()) for p in re.split("[,:]", s)], "files": split_and_match_files, "quickstart_file": expand_path, "junit_xml": expand_path, - # These two are for backwards compatibility - "silent_imports": bool, - "almost_silent": bool, "follow_imports": check_follow_imports, "no_site_packages": bool, "plugins": lambda s: [p.strip() for p in s.split(",")], "always_true": lambda s: [p.strip() for p in s.split(",")], "always_false": lambda s: [p.strip() for p in s.split(",")], - "disable_error_code": lambda s: [p.strip() for p in s.split(",")], - "enable_error_code": lambda s: [p.strip() for p in s.split(",")], + "enable_incomplete_feature": lambda s: [p.strip() for p in s.split(",")], + "disable_error_code": lambda s: validate_codes([p.strip() for p in s.split(",")]), + "enable_error_code": lambda s: validate_codes([p.strip() for p in s.split(",")]), "package_root": lambda s: [p.strip() for p in s.split(",")], "cache_dir": expand_path, "python_executable": expand_path, "strict": bool, "exclude": lambda s: [s.strip()], + "packages": try_split, + "modules": try_split, } # Reuse the ini_config_types and overwrite the diff @@ -161,17 +171,19 @@ def check_follow_imports(choice: str) -> str: toml_config_types.update( { "python_version": parse_version, - "strict_optional_whitelist": try_split, "mypy_path": lambda s: [expand_path(p) for p in try_split(s, "[,:]")], "files": lambda s: split_and_match_files_list(try_split(s)), "follow_imports": lambda s: check_follow_imports(str(s)), "plugins": try_split, "always_true": try_split, "always_false": try_split, - "disable_error_code": try_split, - "enable_error_code": try_split, + "enable_incomplete_feature": try_split, + "disable_error_code": lambda s: validate_codes(try_split(s)), + "enable_error_code": lambda s: validate_codes(try_split(s)), "package_root": try_split, "exclude": str_or_array_as_list, + "packages": try_split, + "modules": try_split, } ) @@ -263,6 +275,7 @@ def parse_config_file( file=stderr, ) updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS} + globs = name[5:] for glob in globs.split(","): # For backwards compatibility, replace (back)slashes with dots. @@ -424,6 +437,9 @@ def parse_section( elif key.startswith("disallow") and hasattr(template, key[3:]): options_key = key[3:] invert = True + elif key.startswith("show_") and hasattr(template, "hide_" + key[5:]): + options_key = "hide_" + key[5:] + invert = True elif key == "strict": pass # Special handling below else: @@ -461,26 +477,14 @@ def parse_section( if v: set_strict_flags() continue - if key == "silent_imports": - print( - "%ssilent_imports has been replaced by " - "ignore_missing_imports=True; follow_imports=skip" % prefix, - file=stderr, - ) - if v: - if "ignore_missing_imports" not in results: - results["ignore_missing_imports"] = True - if "follow_imports" not in results: - results["follow_imports"] = "skip" - if key == "almost_silent": - print( - "%salmost_silent has been replaced by " "follow_imports=error" % prefix, - file=stderr, - ) - if v: - if "follow_imports" not in results: - results["follow_imports"] = "error" results[options_key] = v + + # These two flags act as per-module overrides, so store the empty defaults. + if "disable_error_code" not in results: + results["disable_error_code"] = [] + if "enable_error_code" not in results: + results["enable_error_code"] = [] + return results, report_dirs diff --git a/mypy/constraints.py b/mypy/constraints.py index f9cc68a0a7eb..49b042d5baf0 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -34,6 +34,7 @@ TypeQuery, TypeType, TypeVarId, + TypeVarLikeType, TypeVarTupleType, TypeVarType, TypeVisitor, @@ -53,6 +54,7 @@ extract_unpack, find_unpack_in_list, split_with_instance, + split_with_mapped_and_template, split_with_prefix_and_suffix, ) @@ -73,10 +75,11 @@ class Constraint: op = 0 # SUBTYPE_OF or SUPERTYPE_OF target: Type - def __init__(self, type_var: TypeVarId, op: int, target: Type) -> None: - self.type_var = type_var + def __init__(self, type_var: TypeVarLikeType, op: int, target: Type) -> None: + self.type_var = type_var.id self.op = op self.target = target + self.origin_type_var = type_var def __repr__(self) -> str: op_str = "<:" @@ -108,16 +111,41 @@ def infer_constraints_for_callable( mapper = ArgTypeExpander(context) for i, actuals in enumerate(formal_to_actual): - for actual in actuals: - actual_arg_type = arg_types[actual] - if actual_arg_type is None: - continue + if isinstance(callee.arg_types[i], UnpackType): + unpack_type = callee.arg_types[i] + assert isinstance(unpack_type, UnpackType) + + # In this case we are binding all of the actuals to *args + # and we want a constraint that the typevar tuple being unpacked + # is equal to a type list of all the actuals. + actual_types = [] + for actual in actuals: + actual_arg_type = arg_types[actual] + if actual_arg_type is None: + continue - actual_type = mapper.expand_actual_type( - actual_arg_type, arg_kinds[actual], callee.arg_names[i], callee.arg_kinds[i] - ) - c = infer_constraints(callee.arg_types[i], actual_type, SUPERTYPE_OF) - constraints.extend(c) + actual_types.append( + mapper.expand_actual_type( + actual_arg_type, + arg_kinds[actual], + callee.arg_names[i], + callee.arg_kinds[i], + ) + ) + + assert isinstance(unpack_type.type, TypeVarTupleType) + constraints.append(Constraint(unpack_type.type, SUPERTYPE_OF, TypeList(actual_types))) + else: + for actual in actuals: + actual_arg_type = arg_types[actual] + if actual_arg_type is None: + continue + + actual_type = mapper.expand_actual_type( + actual_arg_type, arg_kinds[actual], callee.arg_names[i], callee.arg_kinds[i] + ) + c = infer_constraints(callee.arg_types[i], actual_type, SUPERTYPE_OF) + constraints.extend(c) return constraints @@ -162,7 +190,6 @@ def infer_constraints(template: Type, actual: Type, direction: int) -> list[Cons def _infer_constraints(template: Type, actual: Type, direction: int) -> list[Constraint]: - orig_template = template template = get_proper_type(template) actual = get_proper_type(actual) @@ -190,7 +217,7 @@ def _infer_constraints(template: Type, actual: Type, direction: int) -> list[Con # T :> U2", but they are not equivalent to the constraint solver, # which never introduces new Union types (it uses join() instead). if isinstance(template, TypeVarType): - return [Constraint(template.id, direction, actual)] + return [Constraint(template, direction, actual)] # Now handle the case of either template or actual being a Union. # For a Union to be a subtype of another type, every item of the Union @@ -286,7 +313,7 @@ def merge_with_any(constraint: Constraint) -> Constraint: # TODO: if we will support multiple sources Any, use this here instead. any_type = AnyType(TypeOfAny.implementation_artifact) return Constraint( - constraint.type_var, + constraint.origin_type_var, constraint.op, UnionType.make_union([target, any_type], target.line, target.column), ) @@ -345,11 +372,37 @@ def any_constraints(options: list[list[Constraint] | None], eager: bool) -> list merged_option = None merged_options.append(merged_option) return any_constraints(list(merged_options), eager) + + # If normal logic didn't work, try excluding trivially unsatisfiable constraint (due to + # upper bounds) from each option, and comparing them again. + filtered_options = [filter_satisfiable(o) for o in options] + if filtered_options != options: + return any_constraints(filtered_options, eager=eager) + # Otherwise, there are either no valid options or multiple, inconsistent valid # options. Give up and deduce nothing. return [] +def filter_satisfiable(option: list[Constraint] | None) -> list[Constraint] | None: + """Keep only constraints that can possibly be satisfied. + + Currently, we filter out constraints where target is not a subtype of the upper bound. + Since those can be never satisfied. We may add more cases in future if it improves type + inference. + """ + if not option: + return option + satisfiable = [] + for c in option: + # TODO: add similar logic for TypeVar values (also in various other places)? + if mypy.subtypes.is_subtype(c.target, c.origin_type_var.upper_bound): + satisfiable.append(c) + if not satisfiable: + return None + return satisfiable + + def is_same_constraints(x: list[Constraint], y: list[Constraint]) -> bool: for c1 in x: if not any(is_same_constraint(c1, c2) for c2 in y): @@ -513,7 +566,33 @@ def visit_instance(self, template: Instance) -> list[Constraint]: template.type.inferring.pop() return res if isinstance(actual, CallableType) and actual.fallback is not None: + if actual.is_type_obj() and template.type.is_protocol: + ret_type = get_proper_type(actual.ret_type) + if isinstance(ret_type, TupleType): + ret_type = mypy.typeops.tuple_fallback(ret_type) + if isinstance(ret_type, Instance): + if self.direction == SUBTYPE_OF: + subtype = template + else: + subtype = ret_type + res.extend( + self.infer_constraints_from_protocol_members( + ret_type, template, subtype, template, class_obj=True + ) + ) actual = actual.fallback + if isinstance(actual, TypeType) and template.type.is_protocol: + if isinstance(actual.item, Instance): + if self.direction == SUBTYPE_OF: + subtype = template + else: + subtype = actual.item + res.extend( + self.infer_constraints_from_protocol_members( + actual.item, template, subtype, template, class_obj=True + ) + ) + if isinstance(actual, Overloaded) and actual.fallback is not None: actual = actual.fallback if isinstance(actual, TypedDictType): @@ -523,15 +602,66 @@ def visit_instance(self, template: Instance) -> list[Constraint]: if isinstance(actual, Instance): instance = actual erased = erase_typevars(template) - assert isinstance(erased, Instance) # type: ignore + assert isinstance(erased, Instance) # type: ignore[misc] # We always try nominal inference if possible, # it is much faster than the structural one. if self.direction == SUBTYPE_OF and template.type.has_base(instance.type.fullname): mapped = map_instance_to_supertype(template, instance.type) tvars = mapped.type.defn.type_vars + + if instance.type.has_type_var_tuple_type: + mapped_prefix, mapped_middle, mapped_suffix = split_with_instance(mapped) + instance_prefix, instance_middle, instance_suffix = split_with_instance( + instance + ) + + # Add a constraint for the type var tuple, and then + # remove it for the case below. + instance_unpack = extract_unpack(instance_middle) + if instance_unpack is not None: + if isinstance(instance_unpack, TypeVarTupleType): + res.append( + Constraint( + instance_unpack, SUBTYPE_OF, TypeList(list(mapped_middle)) + ) + ) + elif ( + isinstance(instance_unpack, Instance) + and instance_unpack.type.fullname == "builtins.tuple" + ): + for item in mapped_middle: + res.extend( + infer_constraints( + instance_unpack.args[0], item, self.direction + ) + ) + elif isinstance(instance_unpack, TupleType): + if len(instance_unpack.items) == len(mapped_middle): + for instance_arg, item in zip( + instance_unpack.items, mapped_middle + ): + res.extend( + infer_constraints(instance_arg, item, self.direction) + ) + + mapped_args = mapped_prefix + mapped_suffix + instance_args = instance_prefix + instance_suffix + + assert instance.type.type_var_tuple_prefix is not None + assert instance.type.type_var_tuple_suffix is not None + tvars_prefix, _, tvars_suffix = split_with_prefix_and_suffix( + tuple(tvars), + instance.type.type_var_tuple_prefix, + instance.type.type_var_tuple_suffix, + ) + tvars = list(tvars_prefix + tvars_suffix) + else: + mapped_args = mapped.args + instance_args = instance.args + # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. - for tvar, mapped_arg, instance_arg in zip(tvars, mapped.args, instance.args): + for tvar, mapped_arg, instance_arg in zip(tvars, mapped_args, instance_args): # TODO(PEP612): More ParamSpec work (or is Parameters the only thing accepted) if isinstance(tvar, TypeVarType): # The constraints for generic type parameters depend on variance. @@ -560,11 +690,12 @@ def visit_instance(self, template: Instance) -> list[Constraint]: suffix.arg_kinds[len(prefix.arg_kinds) :], suffix.arg_names[len(prefix.arg_names) :], ) - res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) + res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix)) elif isinstance(suffix, ParamSpecType): - res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) - elif isinstance(tvar, TypeVarTupleType): - raise NotImplementedError + res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix)) + else: + # This case should have been handled above. + assert not isinstance(tvar, TypeVarTupleType) return res elif self.direction == SUPERTYPE_OF and instance.type.has_base(template.type.fullname): @@ -575,6 +706,16 @@ def visit_instance(self, template: Instance) -> list[Constraint]: template_prefix, template_middle, template_suffix = split_with_instance( template ) + split_result = split_with_mapped_and_template(mapped, template) + assert split_result is not None + ( + mapped_prefix, + mapped_middle, + mapped_suffix, + template_prefix, + template_middle, + template_suffix, + ) = split_result # Add a constraint for the type var tuple, and then # remove it for the case below. @@ -583,18 +724,27 @@ def visit_instance(self, template: Instance) -> list[Constraint]: if isinstance(template_unpack, TypeVarTupleType): res.append( Constraint( - template_unpack.id, SUPERTYPE_OF, TypeList(list(mapped_middle)) + template_unpack, SUPERTYPE_OF, TypeList(list(mapped_middle)) ) ) elif ( isinstance(template_unpack, Instance) and template_unpack.type.fullname == "builtins.tuple" ): - # TODO: check homogenous tuple case - raise NotImplementedError + for item in mapped_middle: + res.extend( + infer_constraints( + template_unpack.args[0], item, self.direction + ) + ) elif isinstance(template_unpack, TupleType): - # TODO: check tuple case - raise NotImplementedError + if len(template_unpack.items) == len(mapped_middle): + for template_arg, item in zip( + template_unpack.items, mapped_middle + ): + res.extend( + infer_constraints(template_arg, item, self.direction) + ) mapped_args = mapped_prefix + mapped_suffix template_args = template_prefix + template_suffix @@ -644,9 +794,12 @@ def visit_instance(self, template: Instance) -> list[Constraint]: suffix.arg_kinds[len(prefix.arg_kinds) :], suffix.arg_names[len(prefix.arg_names) :], ) - res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) + res.append(Constraint(template_arg, SUPERTYPE_OF, suffix)) elif isinstance(suffix, ParamSpecType): - res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) + res.append(Constraint(template_arg, SUPERTYPE_OF, suffix)) + else: + # This case should have been handled above. + assert not isinstance(tvar, TypeVarTupleType) return res if ( template.type.is_protocol @@ -687,6 +840,9 @@ def visit_instance(self, template: Instance) -> list[Constraint]: ) instance.type.inferring.pop() return res + if res: + return res + if isinstance(actual, AnyType): return self.infer_against_any(template.args, actual) if ( @@ -712,7 +868,12 @@ def visit_instance(self, template: Instance) -> list[Constraint]: return [] def infer_constraints_from_protocol_members( - self, instance: Instance, template: Instance, subtype: Type, protocol: Instance + self, + instance: Instance, + template: Instance, + subtype: Type, + protocol: Instance, + class_obj: bool = False, ) -> list[Constraint]: """Infer constraints for situations where either 'template' or 'instance' is a protocol. @@ -722,22 +883,26 @@ def infer_constraints_from_protocol_members( """ res = [] for member in protocol.type.protocol_members: - inst = mypy.subtypes.find_member(member, instance, subtype) + inst = mypy.subtypes.find_member(member, instance, subtype, class_obj=class_obj) temp = mypy.subtypes.find_member(member, template, subtype) if inst is None or temp is None: return [] # See #11020 # The above is safe since at this point we know that 'instance' is a subtype # of (erased) 'template', therefore it defines all protocol members res.extend(infer_constraints(temp, inst, self.direction)) - if mypy.subtypes.IS_SETTABLE in mypy.subtypes.get_member_flags(member, protocol.type): + if mypy.subtypes.IS_SETTABLE in mypy.subtypes.get_member_flags(member, protocol): # Settable members are invariant, add opposite constraints res.extend(infer_constraints(temp, inst, neg_op(self.direction))) return res def visit_callable_type(self, template: CallableType) -> list[Constraint]: + # Normalize callables before matching against each other. + # Note that non-normalized callables can be created in annotations + # using e.g. callback protocols. + template = template.with_unpacked_kwargs() if isinstance(self.actual, CallableType): res: list[Constraint] = [] - cactual = self.actual + cactual = self.actual.with_unpacked_kwargs() param_spec = template.param_spec() if param_spec is None: # FIX verify argument counts @@ -763,7 +928,7 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: prefix_len = min(prefix_len, max_prefix_len) res.append( Constraint( - param_spec.id, + param_spec, SUBTYPE_OF, cactual.copy_modified( arg_types=cactual.arg_types[prefix_len:], @@ -774,7 +939,7 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: ) ) else: - res.append(Constraint(param_spec.id, SUBTYPE_OF, cactual_ps)) + res.append(Constraint(param_spec, SUBTYPE_OF, cactual_ps)) # compare prefixes cactual_prefix = cactual.copy_modified( @@ -805,7 +970,7 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: else: res = [ Constraint( - param_spec.id, + param_spec, SUBTYPE_OF, callable_with_ellipsis(any_type, any_type, template.fallback), ) @@ -877,7 +1042,7 @@ def visit_tuple_type(self, template: TupleType) -> list[Constraint]: modified_actual = actual.copy_modified(items=list(actual_items)) return [ Constraint( - type_var=unpacked_type.id, op=self.direction, target=modified_actual + type_var=unpacked_type, op=self.direction, target=modified_actual ) ] diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py index 8a4027aa8262..efa1b5f01288 100644 --- a/mypy/dmypy/client.py +++ b/mypy/dmypy/client.py @@ -19,7 +19,7 @@ from mypy.dmypy_os import alive, kill from mypy.dmypy_util import DEFAULT_STATUS_FILE, receive from mypy.ipc import IPCClient, IPCException -from mypy.util import check_python_version, get_terminal_width +from mypy.util import check_python_version, get_terminal_width, should_force_color from mypy.version import __version__ # Argument parser. Subparsers are tied to action functions by the @@ -653,7 +653,7 @@ def request( args["command"] = command # Tell the server whether this request was initiated from a human-facing terminal, # so that it can format the type checking output accordingly. - args["is_tty"] = sys.stdout.isatty() or int(os.getenv("MYPY_FORCE_COLOR", "0")) > 0 + args["is_tty"] = sys.stdout.isatty() or should_force_color() args["terminal_width"] = get_terminal_width() bdata = json.dumps(args).encode("utf8") _, name = get_status(status_file) @@ -665,6 +665,10 @@ def request( return {"error": str(err)} # TODO: Other errors, e.g. ValueError, UnicodeError else: + # Display debugging output written to stdout in the server process for convenience. + stdout = response.get("stdout") + if stdout: + sys.stdout.write(stdout) return response diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 4b12bcbe9a29..671999065e7d 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -197,7 +197,7 @@ def __init__(self, options: Options, status_file: str, timeout: int | None = Non # Since the object is created in the parent process we can check # the output terminal options here. - self.formatter = FancyFormatter(sys.stdout, sys.stderr, options.show_error_codes) + self.formatter = FancyFormatter(sys.stdout, sys.stderr, options.hide_error_codes) def _response_metadata(self) -> dict[str, str]: py_version = f"{self.options.python_version[0]}_{self.options.python_version[1]}" @@ -214,6 +214,8 @@ def serve(self) -> None: while True: with server: data = receive(server) + debug_stdout = io.StringIO() + sys.stdout = debug_stdout resp: dict[str, Any] = {} if "command" not in data: resp = {"error": "No command found in request"} @@ -230,8 +232,10 @@ def serve(self) -> None: tb = traceback.format_exception(*sys.exc_info()) resp = {"error": "Daemon crashed!\n" + "".join(tb)} resp.update(self._response_metadata()) + resp["stdout"] = debug_stdout.getvalue() server.write(json.dumps(resp).encode("utf8")) raise + resp["stdout"] = debug_stdout.getvalue() try: resp.update(self._response_metadata()) server.write(json.dumps(resp).encode("utf8")) @@ -267,7 +271,9 @@ def run_command(self, command: str, data: dict[str, object]) -> dict[str, object # Only the above commands use some error formatting. del data["is_tty"] del data["terminal_width"] - return method(self, **data) + ret = method(self, **data) + assert isinstance(ret, dict) + return ret # Command functions (run in the server via RPC). @@ -942,7 +948,7 @@ def cmd_hang(self) -> dict[str, object]: def get_meminfo() -> dict[str, Any]: res: dict[str, Any] = {} try: - import psutil # type: ignore # It's not in typeshed yet + import psutil except ImportError: res["memory_psutil_missing"] = ( "psutil not found, run pip install mypy[dmypy] " diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index 955b30f915b5..f2a74c332b2e 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -33,7 +33,9 @@ def __str__(self) -> str: CALL_OVERLOAD: Final = ErrorCode( "call-overload", "Check that an overload variant matches arguments", "General" ) -VALID_TYPE: Final = ErrorCode("valid-type", "Check that type (annotation) is valid", "General") +VALID_TYPE: Final[ErrorCode] = ErrorCode( + "valid-type", "Check that type (annotation) is valid", "General" +) VAR_ANNOTATED: Final = ErrorCode( "var-annotated", "Require variable annotation if type can't be inferred", "General" ) @@ -46,7 +48,7 @@ def __str__(self) -> str: RETURN_VALUE: Final[ErrorCode] = ErrorCode( "return-value", "Check that return value is compatible with signature", "General" ) -ASSIGNMENT: Final = ErrorCode( +ASSIGNMENT: Final[ErrorCode] = ErrorCode( "assignment", "Check that assigned value is compatible with target", "General" ) TYPE_ARG: Final = ErrorCode("type-arg", "Check that generic type arguments are present", "General") @@ -78,6 +80,9 @@ def __str__(self) -> str: ABSTRACT: Final = ErrorCode( "abstract", "Prevent instantiation of classes with abstract attributes", "General" ) +TYPE_ABSTRACT: Final = ErrorCode( + "type-abstract", "Require only concrete classes where Type[...] is expected", "General" +) VALID_NEWTYPE: Final = ErrorCode( "valid-newtype", "Check that argument 2 to NewType is valid", "General" ) @@ -94,6 +99,16 @@ def __str__(self) -> str: UNUSED_COROUTINE: Final = ErrorCode( "unused-coroutine", "Ensure that all coroutines are used", "General" ) +# TODO: why do we need the explicit type here? Without it mypyc CI builds fail with +# mypy/message_registry.py:37: error: Cannot determine type of "EMPTY_BODY" [has-type] +EMPTY_BODY: Final[ErrorCode] = ErrorCode( + "empty-body", + "A dedicated error code to opt out return errors for empty/trivial bodies", + "General", +) +SAFE_SUPER: Final = ErrorCode( + "safe-super", "Warn about calls to abstract methods with empty/trivial bodies", "General" +) # These error codes aren't enabled by default. NO_UNTYPED_DEF: Final[ErrorCode] = ErrorCode( @@ -122,6 +137,15 @@ def __str__(self) -> str: UNREACHABLE: Final = ErrorCode( "unreachable", "Warn about unreachable statements or expressions", "General" ) +ANNOTATION_UNCHECKED = ErrorCode( + "annotation-unchecked", "Notify about type annotations in unchecked functions", "General" +) +PARTIALLY_DEFINED: Final[ErrorCode] = ErrorCode( + "partially-defined", + "Warn about variables that are defined only in some execution paths", + "General", + default_enabled=False, +) REDUNDANT_EXPR: Final = ErrorCode( "redundant-expr", "Warn about redundant expressions", "General", default_enabled=False ) @@ -131,6 +155,11 @@ def __str__(self) -> str: "General", default_enabled=False, ) +TRUTHY_FUNCTION: Final[ErrorCode] = ErrorCode( + "truthy-function", + "Warn about function that always evaluate to true in boolean contexts", + "General", +) NAME_MATCH: Final = ErrorCode( "name-match", "Check that type definition has consistent naming", "General" ) diff --git a/mypy/errors.py b/mypy/errors.py index 7aa40a235c1e..bfc44a858010 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -17,6 +17,9 @@ T = TypeVar("T") +# Show error codes for some note-level messages (these usually appear alone +# and not as a comment for a previous error-level message). +SHOW_NOTE_CODES: Final = {codes.ANNOTATION_UNCHECKED} allowed_duplicates: Final = ["@overload", "Got:", "Expected:"] # Keep track of the original error code when the error code of a message is changed. @@ -257,18 +260,17 @@ def __init__( self, show_error_context: bool = False, show_column_numbers: bool = False, - show_error_codes: bool = False, + hide_error_codes: bool = False, pretty: bool = False, show_error_end: bool = False, read_source: Callable[[str], list[str] | None] | None = None, show_absolute_path: bool = False, - enabled_error_codes: set[ErrorCode] | None = None, - disabled_error_codes: set[ErrorCode] | None = None, many_errors_threshold: int = -1, + options: Options | None = None, ) -> None: self.show_error_context = show_error_context self.show_column_numbers = show_column_numbers - self.show_error_codes = show_error_codes + self.hide_error_codes = hide_error_codes self.show_absolute_path = show_absolute_path self.pretty = pretty self.show_error_end = show_error_end @@ -276,9 +278,8 @@ def __init__( assert show_column_numbers, "Inconsistent formatting, must be prevented by argparse" # We use fscache to read source code when showing snippets. self.read_source = read_source - self.enabled_error_codes = enabled_error_codes or set() - self.disabled_error_codes = disabled_error_codes or set() self.many_errors_threshold = many_errors_threshold + self.options = options self.initialize() def initialize(self) -> None: @@ -313,7 +314,9 @@ def simplify_path(self, file: str) -> str: file = os.path.normpath(file) return remove_path_prefix(file, self.ignore_prefix) - def set_file(self, file: str, module: str | None, scope: Scope | None = None) -> None: + def set_file( + self, file: str, module: str | None, options: Options, scope: Scope | None = None + ) -> None: """Set the path and module id of the current file.""" # The path will be simplified later, in render_messages. That way # * 'file' is always a key that uniquely identifies a source file @@ -324,6 +327,7 @@ def set_file(self, file: str, module: str | None, scope: Scope | None = None) -> self.file = file self.target_module = module self.scope = scope + self.options = options def set_file_ignored_lines( self, file: str, ignored_lines: dict[int, list[str]], ignore_all: bool = False @@ -586,9 +590,16 @@ def is_ignored_error(self, line: int, info: ErrorInfo, ignores: dict[int, list[s return False def is_error_code_enabled(self, error_code: ErrorCode) -> bool: - if error_code in self.disabled_error_codes: + if self.options: + current_mod_disabled = self.options.disabled_error_codes + current_mod_enabled = self.options.enabled_error_codes + else: + current_mod_disabled = set() + current_mod_enabled = set() + + if error_code in current_mod_disabled: return False - elif error_code in self.enabled_error_codes: + elif error_code in current_mod_enabled: return True else: return error_code.default_enabled @@ -609,7 +620,10 @@ def clear_errors_in_targets(self, path: str, targets: set[str]) -> None: self.has_blockers.remove(path) def generate_unused_ignore_errors(self, file: str) -> None: - if is_typeshed_file(file) or file in self.ignored_files: + if ( + is_typeshed_file(self.options.abs_custom_typeshed_dir if self.options else None, file) + or file in self.ignored_files + ): return ignored_lines = self.ignored_lines[file] used_ignored_lines = self.used_ignored_lines[file] @@ -650,7 +664,10 @@ def generate_unused_ignore_errors(self, file: str) -> None: def generate_ignore_without_code_errors( self, file: str, is_warning_unused_ignores: bool ) -> None: - if is_typeshed_file(file) or file in self.ignored_files: + if ( + is_typeshed_file(self.options.abs_custom_typeshed_dir if self.options else None, file) + or file in self.ignored_files + ): return used_ignored_lines = self.used_ignored_lines[file] @@ -768,7 +785,11 @@ def format_messages( s = f"{srcloc}: {severity}: {message}" else: s = message - if self.show_error_codes and code and severity != "note": + if ( + not self.hide_error_codes + and code + and (severity != "note" or code in SHOW_NOTE_CODES) + ): # If note has an error code, it is related to a previous error. Avoid # displaying duplicate error codes. s = f"{s} [{code.code}]" diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 959983ae66d1..08bc216689fb 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,7 +1,8 @@ from __future__ import annotations -from typing import Iterable, Mapping, Sequence, TypeVar, cast +from typing import Iterable, Mapping, Sequence, TypeVar, cast, overload +from mypy.nodes import ARG_STAR from mypy.types import ( AnyType, CallableType, @@ -37,18 +38,36 @@ from mypy.typevartuples import split_with_instance, split_with_prefix_and_suffix +@overload +def expand_type(typ: ProperType, env: Mapping[TypeVarId, Type]) -> ProperType: + ... + + +@overload +def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: + ... + + def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: """Substitute any type variable references in a type given by a type environment. """ - # TODO: use an overloaded signature? (ProperType stays proper after expansion.) return typ.accept(ExpandTypeVisitor(env)) +@overload +def expand_type_by_instance(typ: ProperType, instance: Instance) -> ProperType: + ... + + +@overload +def expand_type_by_instance(typ: Type, instance: Instance) -> Type: + ... + + def expand_type_by_instance(typ: Type, instance: Instance) -> Type: """Substitute type variables in type using values from an Instance. Type variables are considered to be bound by the class declaration.""" - # TODO: use an overloaded signature? (ProperType stays proper after expansion.) if not instance.args: return typ else: @@ -87,7 +106,6 @@ def freshen_function_type_vars(callee: F) -> F: tvs = [] tvmap: dict[TypeVarId, Type] = {} for v in callee.variables: - # TODO(PEP612): fix for ParamSpecType if isinstance(v, TypeVarType): tv: TypeVarLikeType = TypeVarType.new_unification_variable(v) elif isinstance(v, TypeVarTupleType): @@ -136,7 +154,7 @@ def visit_erased_type(self, t: ErasedType) -> Type: def visit_instance(self, t: Instance) -> Type: args = self.expand_types_with_unpack(list(t.args)) if isinstance(args, list): - return Instance(t.type, args, t.line, t.column) + return t.copy_modified(args=args) else: return args @@ -196,31 +214,7 @@ def visit_unpack_type(self, t: UnpackType) -> Type: assert False, "Mypy bug: unpacking must happen at a higher level" def expand_unpack(self, t: UnpackType) -> list[Type] | Instance | AnyType | None: - """May return either a list of types to unpack to, any, or a single - variable length tuple. The latter may not be valid in all contexts. - """ - if isinstance(t.type, TypeVarTupleType): - repl = get_proper_type(self.variables.get(t.type.id, t)) - if isinstance(repl, TupleType): - return repl.items - if isinstance(repl, TypeList): - return repl.items - elif isinstance(repl, Instance) and repl.type.fullname == "builtins.tuple": - return repl - elif isinstance(repl, AnyType): - # tuple[Any, ...] would be better, but we don't have - # the type info to construct that type here. - return repl - elif isinstance(repl, TypeVarTupleType): - return [UnpackType(typ=repl)] - elif isinstance(repl, UnpackType): - return [repl] - elif isinstance(repl, UninhabitedType): - return None - else: - raise NotImplementedError(f"Invalid type replacement to expand: {repl}") - else: - raise NotImplementedError(f"Invalid type to expand: {t.type}") + return expand_unpack_with_variables(t, self.variables) def visit_parameters(self, t: Parameters) -> Type: return t.copy_modified(arg_types=self.expand_types(t.arg_types)) @@ -250,8 +244,23 @@ def visit_callable_type(self, t: CallableType) -> Type: type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), ) + var_arg = t.var_arg() + if var_arg is not None and isinstance(var_arg.typ, UnpackType): + expanded = self.expand_unpack(var_arg.typ) + # Handle other cases later. + assert isinstance(expanded, list) + assert len(expanded) == 1 and isinstance(expanded[0], UnpackType) + star_index = t.arg_kinds.index(ARG_STAR) + arg_types = ( + self.expand_types(t.arg_types[:star_index]) + + expanded + + self.expand_types(t.arg_types[star_index + 1 :]) + ) + else: + arg_types = self.expand_types(t.arg_types) + return t.copy_modified( - arg_types=self.expand_types(t.arg_types), + arg_types=arg_types, ret_type=t.ret_type.accept(self), type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), ) @@ -344,3 +353,33 @@ def expand_types(self, types: Iterable[Type]) -> list[Type]: for t in types: a.append(t.accept(self)) return a + + +def expand_unpack_with_variables( + t: UnpackType, variables: Mapping[TypeVarId, Type] +) -> list[Type] | Instance | AnyType | None: + """May return either a list of types to unpack to, any, or a single + variable length tuple. The latter may not be valid in all contexts. + """ + if isinstance(t.type, TypeVarTupleType): + repl = get_proper_type(variables.get(t.type.id, t)) + if isinstance(repl, TupleType): + return repl.items + if isinstance(repl, TypeList): + return repl.items + elif isinstance(repl, Instance) and repl.type.fullname == "builtins.tuple": + return repl + elif isinstance(repl, AnyType): + # tuple[Any, ...] would be better, but we don't have + # the type info to construct that type here. + return repl + elif isinstance(repl, TypeVarTupleType): + return [UnpackType(typ=repl)] + elif isinstance(repl, UnpackType): + return [repl] + elif isinstance(repl, UninhabitedType): + return None + else: + raise NotImplementedError(f"Invalid type replacement to expand: {repl}") + else: + raise NotImplementedError(f"Invalid type to expand: {t.type}") diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 2f749af6a467..209ebb89f36b 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -212,6 +212,10 @@ def ast3_parse( MatchAs = Any MatchOr = Any AstNode = Union[ast3.expr, ast3.stmt, ast3.ExceptHandler] + if sys.version_info >= (3, 11): + TryStar = ast3.TryStar + else: + TryStar = Any except ImportError: try: from typed_ast import ast35 # type: ignore[attr-defined] # noqa: F401 @@ -258,15 +262,17 @@ def parse( on failure. Otherwise, use the errors object to report parse errors. """ raise_on_error = False - if errors is None: - errors = Errors() - raise_on_error = True if options is None: options = Options() - errors.set_file(fnam, module) + if errors is None: + errors = Errors(hide_error_codes=options.hide_error_codes) + raise_on_error = True + errors.set_file(fnam, module, options=options) is_stub_file = fnam.endswith(".pyi") if is_stub_file: feature_version = defaults.PYTHON3_VERSION[1] + if options.python_version[0] == 3 and options.python_version[1] > feature_version: + feature_version = options.python_version[1] else: assert options.python_version[0] >= 3 feature_version = options.python_version[1] @@ -303,6 +309,7 @@ def parse( if raise_on_error and errors.is_errors(): errors.raise_error() + assert isinstance(tree, MypyFile) return tree @@ -480,6 +487,16 @@ def translate_stmt_list( and self.type_ignores and min(self.type_ignores) < self.get_lineno(stmts[0]) ): + if self.type_ignores[min(self.type_ignores)]: + self.fail( + ( + "type ignore with error code is not supported for modules; " + "use `# mypy: disable-error-code=...`" + ), + line=min(self.type_ignores), + column=0, + blocker=False, + ) self.errors.used_ignored_lines[self.errors.file][min(self.type_ignores)].append( codes.FILE.code ) @@ -731,7 +748,7 @@ def _check_ifstmt_for_overloads( if stmt.else_body is None: return overload_name - if isinstance(stmt.else_body, Block) and len(stmt.else_body.body) == 1: + if len(stmt.else_body.body) == 1: # For elif: else_body contains an IfStmt itself -> do a recursive check. if ( isinstance(stmt.else_body.body[0], (Decorator, FuncDef, OverloadedFuncDef)) @@ -828,7 +845,7 @@ def visit_Module(self, mod: ast3.Module) -> MypyFile: if parsed is not None: self.type_ignores[ti.lineno] = parsed else: - self.fail(INVALID_TYPE_IGNORE, ti.lineno, -1) + self.fail(INVALID_TYPE_IGNORE, ti.lineno, -1, blocker=False) body = self.fix_function_overloads(self.translate_stmt_list(mod.body, ismodule=True)) return MypyFile(body, self.imports, False, self.type_ignores) @@ -888,13 +905,11 @@ def do_func_def( # PEP 484 disallows both type annotations and type comments if n.returns or any(a.type_annotation is not None for a in args): self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset) - translated_args = TypeConverter( + translated_args: list[Type] = TypeConverter( self.errors, line=lineno, override_column=n.col_offset ).translate_expr_list(func_type_ast.argtypes) - arg_types = [ - a if a is not None else AnyType(TypeOfAny.unannotated) - for a in translated_args - ] + # Use a cast to work around `list` invariance + arg_types = cast(List[Optional[Type]], translated_args) return_type = TypeConverter(self.errors, line=lineno).visit(func_type_ast.returns) # add implicit self type @@ -923,7 +938,7 @@ def do_func_def( if any(arg_types) or return_type: if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types): self.fail( - "Ellipses cannot accompany other argument types " "in function type signature", + "Ellipses cannot accompany other argument types in function type signature", lineno, n.col_offset, ) @@ -992,7 +1007,7 @@ def do_func_def( return retval def set_type_optional(self, type: Type | None, initializer: Expression | None) -> None: - if self.options.no_implicit_optional: + if not self.options.implicit_optional: return # Indicate that type should be wrapped in an Optional if arg is initialized to None. optional = isinstance(initializer, NameExpr) and initializer.name == "None" @@ -1238,6 +1253,24 @@ def visit_Try(self, n: ast3.Try) -> TryStmt: ) return self.set_line(node, n) + def visit_TryStar(self, n: TryStar) -> TryStmt: + vs = [ + self.set_line(NameExpr(h.name), h) if h.name is not None else None for h in n.handlers + ] + types = [self.visit(h.type) for h in n.handlers] + handlers = [self.as_required_block(h.body, h.lineno) for h in n.handlers] + + node = TryStmt( + self.as_required_block(n.body, n.lineno), + vs, + types, + handlers, + self.as_block(n.orelse, n.lineno), + self.as_block(n.finalbody, n.lineno), + ) + node.is_star = True + return self.set_line(node, n) + # Assert(expr test, expr? msg) def visit_Assert(self, n: ast3.Assert) -> AssertStmt: node = AssertStmt(self.visit(n.test), self.visit(n.msg)) @@ -1630,7 +1663,9 @@ def visit_ExtSlice(self, n: ast3.ExtSlice) -> TupleExpr: # Index(expr value) def visit_Index(self, n: Index) -> Node: # cast for mypyc's benefit on Python 3.9 - return self.visit(cast(Any, n).value) + value = self.visit(cast(Any, n).value) + assert isinstance(value, Node) + return value # Match(expr subject, match_case* cases) # python 3.10 and later def visit_Match(self, n: Match) -> MatchStmt: @@ -1760,7 +1795,9 @@ def visit(self, node: AST | None) -> ProperType | None: method = "visit_" + node.__class__.__name__ visitor = getattr(self, method, None) if visitor is not None: - return visitor(node) + typ = visitor(node) + assert isinstance(typ, ProperType) + return typ else: return self.invalid_type(node) finally: @@ -1947,7 +1984,9 @@ def visit_Bytes(self, n: Bytes) -> Type: def visit_Index(self, n: ast3.Index) -> Type: # cast for mypyc's benefit on Python 3.9 - return self.visit(cast(Any, n).value) + value = self.visit(cast(Any, n).value) + assert isinstance(value, Type) + return value def visit_Slice(self, n: ast3.Slice) -> Type: return self.invalid_type(n, note="did you mean to use ',' instead of ':' ?") @@ -1971,9 +2010,10 @@ def visit_Subscript(self, n: ast3.Subscript) -> Type: for s in dims: if getattr(s, "col_offset", None) is None: if isinstance(s, ast3.Index): - s.col_offset = s.value.col_offset # type: ignore + s.col_offset = s.value.col_offset # type: ignore[attr-defined] elif isinstance(s, ast3.Slice): - s.col_offset = s.lower.col_offset # type: ignore + assert s.lower is not None + s.col_offset = s.lower.col_offset # type: ignore[attr-defined] sliceval = ast3.Tuple(dims, n.ctx) empty_tuple_index = False diff --git a/mypy/fixup.py b/mypy/fixup.py index 7f7c3129005c..b3a2d43d6b4d 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -13,10 +13,12 @@ FuncDef, MypyFile, OverloadedFuncDef, + ParamSpecExpr, SymbolTable, TypeAlias, TypeInfo, TypeVarExpr, + TypeVarTupleExpr, Var, ) from mypy.types import ( @@ -164,6 +166,12 @@ def visit_type_var_expr(self, tv: TypeVarExpr) -> None: value.accept(self.type_fixer) tv.upper_bound.accept(self.type_fixer) + def visit_paramspec_expr(self, p: ParamSpecExpr) -> None: + p.upper_bound.accept(self.type_fixer) + + def visit_type_var_tuple_expr(self, tv: TypeVarTupleExpr) -> None: + tv.upper_bound.accept(self.type_fixer) + def visit_var(self, v: Var) -> None: if self.current_info is not None: v.info = self.current_info diff --git a/mypy/ipc.py b/mypy/ipc.py index 8e693169ab36..d52769bdb2b1 100644 --- a/mypy/ipc.py +++ b/mypy/ipc.py @@ -252,7 +252,7 @@ def __exit__( # Wait for the client to finish reading the last write before disconnecting if not FlushFileBuffers(self.connection): raise IPCException( - "Failed to flush NamedPipe buffer," "maybe the client hung up?" + "Failed to flush NamedPipe buffer, maybe the client hung up?" ) finally: DisconnectNamedPipe(self.connection) @@ -270,4 +270,6 @@ def connection_name(self) -> str: if sys.platform == "win32": return self.name else: - return self.sock.getsockname() + name = self.sock.getsockname() + assert isinstance(name, str) + return name diff --git a/mypy/join.py b/mypy/join.py index 123488c54ef6..d54febd7462a 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import overload + import mypy.typeops from mypy.maptype import map_instance_to_supertype from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT @@ -131,7 +133,6 @@ def join_instances_via_supertype(self, t: Instance, s: Instance) -> ProperType: best = res assert best is not None for promote in t.type._promote: - promote = get_proper_type(promote) if isinstance(promote, Instance): res = self.join_instances(promote, s) if is_better(res, best): @@ -141,7 +142,7 @@ def join_instances_via_supertype(self, t: Instance, s: Instance) -> ProperType: def join_simple(declaration: Type | None, s: Type, t: Type) -> ProperType: """Return a simple least upper bound given the declared type.""" - # TODO: check infinite recursion for aliases here. + # TODO: check infinite recursion for aliases here? declaration = get_proper_type(declaration) s = get_proper_type(s) t = get_proper_type(t) @@ -172,6 +173,9 @@ def join_simple(declaration: Type | None, s: Type, t: Type) -> ProperType: if isinstance(s, UninhabitedType) and not isinstance(t, UninhabitedType): s, t = t, s + # Meets/joins require callable type normalization. + s, t = normalize_callables(s, t) + value = t.accept(TypeJoinVisitor(s)) if declaration is None or is_subtype(value, declaration): return value @@ -179,17 +183,29 @@ def join_simple(declaration: Type | None, s: Type, t: Type) -> ProperType: return declaration -def trivial_join(s: Type, t: Type) -> ProperType: +def trivial_join(s: Type, t: Type) -> Type: """Return one of types (expanded) if it is a supertype of other, otherwise top type.""" if is_subtype(s, t): - return get_proper_type(t) + return t elif is_subtype(t, s): - return get_proper_type(s) + return s else: return object_or_any_from_type(get_proper_type(t)) -def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) -> ProperType: +@overload +def join_types( + s: ProperType, t: ProperType, instance_joiner: InstanceJoiner | None = None +) -> ProperType: + ... + + +@overload +def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) -> Type: + ... + + +def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) -> Type: """Return the least upper bound of s and t. For example, the join of 'int' and 'object' is 'object'. @@ -229,6 +245,9 @@ def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) elif isinstance(t, PlaceholderType): return AnyType(TypeOfAny.from_error) + # Meets/joins require callable type normalization. + s, t = normalize_callables(s, t) + # Use a visitor to handle non-trivial cases. return t.accept(TypeJoinVisitor(s, instance_joiner)) @@ -437,7 +456,7 @@ def visit_tuple_type(self, t: TupleType) -> ProperType: if self.s.length() == t.length(): items: list[Type] = [] for i in range(t.length()): - items.append(self.join(t.items[i], self.s.items[i])) + items.append(join_types(t.items[i], self.s.items[i])) return TupleType(items, fallback) else: return fallback @@ -481,7 +500,7 @@ def visit_partial_type(self, t: PartialType) -> ProperType: def visit_type_type(self, t: TypeType) -> ProperType: if isinstance(self.s, TypeType): - return TypeType.make_normalized(self.join(t.item, self.s.item), line=t.line) + return TypeType.make_normalized(join_types(t.item, self.s.item), line=t.line) elif isinstance(self.s, Instance) and self.s.type.fullname == "builtins.type": return self.s else: @@ -490,9 +509,6 @@ def visit_type_type(self, t: TypeType) -> ProperType: def visit_type_alias_type(self, t: TypeAliasType) -> ProperType: assert False, f"This should be never called, got {t}" - def join(self, s: Type, t: Type) -> ProperType: - return join_types(s, t) - def default(self, typ: Type) -> ProperType: typ = get_proper_type(typ) if isinstance(typ, Instance): @@ -528,6 +544,14 @@ def is_better(t: Type, s: Type) -> bool: return False +def normalize_callables(s: ProperType, t: ProperType) -> tuple[ProperType, ProperType]: + if isinstance(s, (CallableType, Overloaded)): + s = s.with_unpacked_kwargs() + if isinstance(t, (CallableType, Overloaded)): + t = t.with_unpacked_kwargs() + return s, t + + def is_similar_callables(t: CallableType, s: CallableType) -> bool: """Return True if t and s have identical numbers of arguments, default arguments and varargs. @@ -545,10 +569,10 @@ def join_similar_callables(t: CallableType, s: CallableType) -> CallableType: arg_types: list[Type] = [] for i in range(len(t.arg_types)): arg_types.append(meet_types(t.arg_types[i], s.arg_types[i])) - # TODO in combine_similar_callables also applies here (names and kinds) - # The fallback type can be either 'function' or 'type'. The result should have 'type' as - # fallback only if both operands have it as 'type'. - if t.fallback.type.fullname != "builtins.type": + # TODO in combine_similar_callables also applies here (names and kinds; user metaclasses) + # The fallback type can be either 'function', 'type', or some user-provided metaclass. + # The result should always use 'function' as a fallback if either operands are using it. + if t.fallback.type.fullname == "builtins.function": fallback = t.fallback else: fallback = s.fallback @@ -566,9 +590,10 @@ def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: for i in range(len(t.arg_types)): arg_types.append(join_types(t.arg_types[i], s.arg_types[i])) # TODO kinds and argument names - # The fallback type can be either 'function' or 'type'. The result should have 'type' as - # fallback only if both operands have it as 'type'. - if t.fallback.type.fullname != "builtins.type": + # TODO what should happen if one fallback is 'type' and the other is a user-provided metaclass? + # The fallback type can be either 'function', 'type', or some user-provided metaclass. + # The result should always use 'function' as a fallback if either operands are using it. + if t.fallback.type.fullname == "builtins.function": fallback = t.fallback else: fallback = s.fallback @@ -639,19 +664,19 @@ def object_or_any_from_type(typ: ProperType) -> ProperType: return AnyType(TypeOfAny.implementation_artifact) -def join_type_list(types: list[Type]) -> ProperType: +def join_type_list(types: list[Type]) -> Type: if not types: # This is a little arbitrary but reasonable. Any empty tuple should be compatible # with all variable length tuples, and this makes it possible. return UninhabitedType() - joined = get_proper_type(types[0]) + joined = types[0] for t in types[1:]: joined = join_types(joined, t) return joined -def unpack_callback_protocol(t: Instance) -> Type | None: +def unpack_callback_protocol(t: Instance) -> ProperType | None: assert t.type.is_protocol if t.type.protocol_members == ["__call__"]: - return find_member("__call__", t, t, is_operator=True) + return get_proper_type(find_member("__call__", t, t, is_operator=True)) return None diff --git a/mypy/literals.py b/mypy/literals.py index 43425755aae8..9d91cf728b06 100644 --- a/mypy/literals.py +++ b/mypy/literals.py @@ -173,7 +173,7 @@ def visit_op_expr(self, e: OpExpr) -> Key: return ("Binary", e.op, literal_hash(e.left), literal_hash(e.right)) def visit_comparison_expr(self, e: ComparisonExpr) -> Key: - rest: Any = tuple(e.operators) + rest: tuple[str | Key | None, ...] = tuple(e.operators) rest += tuple(literal_hash(o) for o in e.operands) return ("Comparison",) + rest @@ -182,7 +182,7 @@ def visit_unary_expr(self, e: UnaryExpr) -> Key: def seq_expr(self, e: ListExpr | TupleExpr | SetExpr, name: str) -> Key | None: if all(literal(x) == LITERAL_YES for x in e.items): - rest: Any = tuple(literal_hash(x) for x in e.items) + rest: tuple[Key | None, ...] = tuple(literal_hash(x) for x in e.items) return (name,) + rest return None @@ -191,7 +191,7 @@ def visit_list_expr(self, e: ListExpr) -> Key | None: def visit_dict_expr(self, e: DictExpr) -> Key | None: if all(a and literal(a) == literal(b) == LITERAL_YES for a, b in e.items): - rest: Any = tuple( + rest: tuple[Key | None, ...] = tuple( (literal_hash(a) if a else None, literal_hash(b)) for a, b in e.items ) return ("Dict",) + rest diff --git a/mypy/main.py b/mypy/main.py index 7388e9a375ff..405596c20991 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -18,7 +18,7 @@ from mypy.find_sources import InvalidSourceList, create_source_list from mypy.fscache import FileSystemCache from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path -from mypy.options import BuildType, Options +from mypy.options import INCOMPLETE_FEATURES, BuildType, Options from mypy.split_namespace import SplitNamespace from mypy.version import __version__ @@ -67,7 +67,7 @@ def main( if clean_exit: options.fast_exit = False - formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) + formatter = util.FancyFormatter(stdout, stderr, options.hide_error_codes) if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): # Since --install-types performs user input, we want regular stdout and stderr. @@ -110,10 +110,10 @@ def main( print_memory_profile() code = 0 - if messages: + n_errors, n_notes, n_files = util.count_stats(messages) + if messages and n_notes < len(messages): code = 2 if blockers else 1 if options.error_summary: - n_errors, n_notes, n_files = util.count_stats(messages) if n_errors: summary = formatter.format_error( n_errors, n_files, len(sources), blockers=blockers, use_color=options.color_output @@ -151,7 +151,7 @@ def run_build( stdout: TextIO, stderr: TextIO, ) -> tuple[build.BuildResult | None, list[str], bool]: - formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) + formatter = util.FancyFormatter(stdout, stderr, options.hide_error_codes) messages = [] @@ -544,8 +544,9 @@ def add_invertible_flag( title="Import discovery", description="Configure how imports are discovered and followed." ) add_invertible_flag( - "--namespace-packages", - default=False, + "--no-namespace-packages", + dest="namespace_packages", + default=True, help="Support namespace packages (PEP 420, __init__.py-less)", group=imports_group, ) @@ -657,7 +658,7 @@ def add_invertible_flag( "--disallow-any-generics", default=False, strict_flag=True, - help="Disallow usage of generic types that do not specify explicit type " "parameters", + help="Disallow usage of generic types that do not specify explicit type parameters", group=disallow_any_group, ) add_invertible_flag( @@ -720,10 +721,9 @@ def add_invertible_flag( "https://mypy.readthedocs.io/en/stable/kinds_of_types.html#no-strict-optional", ) add_invertible_flag( - "--no-implicit-optional", + "--implicit-optional", default=False, - strict_flag=True, - help="Don't assume arguments with default values of None are Optional", + help="Assume arguments with default values of None are Optional", group=none_group, ) none_group.add_argument("--strict-optional", action="store_true", help=argparse.SUPPRESS) @@ -733,9 +733,6 @@ def add_invertible_flag( dest="strict_optional", help="Disable strict Optional checks (inverse: --strict-optional)", ) - none_group.add_argument( - "--strict-optional-whitelist", metavar="GLOB", nargs="*", help=argparse.SUPPRESS - ) lint_group = parser.add_argument_group( title="Configuring warnings", @@ -874,9 +871,9 @@ def add_invertible_flag( group=error_group, ) add_invertible_flag( - "--show-error-codes", + "--hide-error-codes", default=False, - help="Show error codes in error messages", + help="Hide error codes in error messages", group=error_group, ) add_invertible_flag( @@ -978,9 +975,19 @@ def add_invertible_flag( help="Use a custom typing module", ) internals_group.add_argument( - "--enable-recursive-aliases", + "--disable-recursive-aliases", action="store_true", - help="Experimental support for recursive type aliases", + help="Disable experimental support for recursive type aliases", + ) + # Deprecated reverse variant of the above. + internals_group.add_argument( + "--enable-recursive-aliases", action="store_true", help=argparse.SUPPRESS + ) + parser.add_argument( + "--enable-incomplete-feature", + action="append", + metavar="FEATURE", + help="Enable support of incomplete/experimental features for early preview", ) internals_group.add_argument( "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR" @@ -1002,6 +1009,11 @@ def add_invertible_flag( "the contents of SHADOW_FILE instead.", ) add_invertible_flag("--fast-exit", default=True, help=argparse.SUPPRESS, group=internals_group) + # This flag is useful for mypy tests, where function bodies may be omitted. Plugin developers + # may want to use this as well in their tests. + add_invertible_flag( + "--allow-empty-bodies", default=False, help=argparse.SUPPRESS, group=internals_group + ) report_group = parser.add_argument_group( title="Report generation", description="Generate a report in the specified format." @@ -1105,9 +1117,16 @@ def add_invertible_flag( parser.add_argument( "--cache-map", nargs="+", dest="special-opts:cache_map", help=argparse.SUPPRESS ) + # This one is deprecated, but we will keep it for few releases. parser.add_argument( "--enable-incomplete-features", action="store_true", help=argparse.SUPPRESS ) + parser.add_argument( + "--disable-bytearray-promotion", action="store_true", help=argparse.SUPPRESS + ) + parser.add_argument( + "--disable-memoryview-promotion", action="store_true", help=argparse.SUPPRESS + ) # options specifying code to check code_group = parser.add_argument_group( @@ -1220,8 +1239,13 @@ def set_strict_flags() -> None: # Paths listed in the config file will be ignored if any paths, modules or packages # are passed on the command line. - if options.files and not (special_opts.files or special_opts.packages or special_opts.modules): - special_opts.files = options.files + if not (special_opts.files or special_opts.packages or special_opts.modules): + if options.files: + special_opts.files = options.files + if options.packages: + special_opts.packages = options.packages + if options.modules: + special_opts.modules = options.modules # Check for invalid argument combinations. if require_targets: @@ -1267,10 +1291,22 @@ def set_strict_flags() -> None: # Enabling an error code always overrides disabling options.disabled_error_codes -= options.enabled_error_codes + # Validate incomplete features. + for feature in options.enable_incomplete_feature: + if feature not in INCOMPLETE_FEATURES: + parser.error(f"Unknown incomplete feature: {feature}") + if options.enable_incomplete_features: + print( + "Warning: --enable-incomplete-features is deprecated, use" + " --enable-incomplete-feature=FEATURE instead" + ) + options.enable_incomplete_feature = list(INCOMPLETE_FEATURES) + + # Compute absolute path for custom typeshed (if present). + if options.custom_typeshed_dir is not None: + options.abs_custom_typeshed_dir = os.path.abspath(options.custom_typeshed_dir) + # Set build flags. - if options.strict_optional_whitelist is not None: - # TODO: Deprecate, then kill this flag - options.strict_optional = True if special_opts.find_occurrences: state.find_occurrences = special_opts.find_occurrences.split(".") assert state.find_occurrences is not None @@ -1310,6 +1346,12 @@ def set_strict_flags() -> None: if options.logical_deps: options.cache_fine_grained = True + if options.enable_recursive_aliases: + print( + "Warning: --enable-recursive-aliases is deprecated;" + " recursive types are enabled by default" + ) + # Set target. if special_opts.modules + special_opts.packages: options.build_type = BuildType.MODULE diff --git a/mypy/maptype.py b/mypy/maptype.py index 2cec20a03189..cae904469fed 100644 --- a/mypy/maptype.py +++ b/mypy/maptype.py @@ -1,19 +1,8 @@ from __future__ import annotations -import mypy.typeops from mypy.expandtype import expand_type from mypy.nodes import TypeInfo -from mypy.types import ( - AnyType, - Instance, - ProperType, - TupleType, - Type, - TypeOfAny, - TypeVarId, - get_proper_type, - has_type_vars, -) +from mypy.types import AnyType, Instance, TupleType, Type, TypeOfAny, TypeVarId, has_type_vars def map_instance_to_supertype(instance: Instance, superclass: TypeInfo) -> Instance: @@ -37,8 +26,11 @@ def map_instance_to_supertype(instance: Instance, superclass: TypeInfo) -> Insta # Unfortunately we can't support this for generic recursive tuples. # If we skip this special casing we will fall back to tuple[Any, ...]. env = instance_to_type_environment(instance) - tuple_type = get_proper_type(expand_type(instance.type.tuple_type, env)) + tuple_type = expand_type(instance.type.tuple_type, env) if isinstance(tuple_type, TupleType): + # Make the import here to avoid cyclic imports. + import mypy.typeops + return mypy.typeops.tuple_fallback(tuple_type) if not superclass.type_vars: @@ -101,7 +93,6 @@ def map_instance_to_direct_supertypes(instance: Instance, supertype: TypeInfo) - if b.type == supertype: env = instance_to_type_environment(instance) t = expand_type(b, env) - assert isinstance(t, ProperType) assert isinstance(t, Instance) result.append(t) diff --git a/mypy/meet.py b/mypy/meet.py index 21637f57f233..3e772419ef3e 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -6,7 +6,13 @@ from mypy.erasetype import erase_type from mypy.maptype import map_instance_to_supertype from mypy.state import state -from mypy.subtypes import is_callable_compatible, is_equivalent, is_proper_subtype, is_subtype +from mypy.subtypes import ( + is_callable_compatible, + is_equivalent, + is_proper_subtype, + is_same_type, + is_subtype, +) from mypy.typeops import is_recursive_pair, make_simplified_union, tuple_fallback from mypy.types import ( AnyType, @@ -61,11 +67,25 @@ def meet_types(s: Type, t: Type) -> ProperType: """Return the greatest lower bound of two types.""" if is_recursive_pair(s, t): # This case can trigger an infinite recursion, general support for this will be - # tricky so we use a trivial meet (like for protocols). + # tricky, so we use a trivial meet (like for protocols). return trivial_meet(s, t) s = get_proper_type(s) t = get_proper_type(t) + if isinstance(s, Instance) and isinstance(t, Instance) and s.type == t.type: + # Code in checker.py should merge any extra_items where possible, so we + # should have only compatible extra_items here. We check this before + # the below subtype check, so that extra_attrs will not get erased. + if is_same_type(s, t) and (s.extra_attrs or t.extra_attrs): + if s.extra_attrs and t.extra_attrs: + if len(s.extra_attrs.attrs) > len(t.extra_attrs.attrs): + # Return the one that has more precise information. + return s + return t + if s.extra_attrs: + return s + return t + if not isinstance(s, UnboundType) and not isinstance(t, UnboundType): if is_proper_subtype(s, t, ignore_promotions=True): return s @@ -78,6 +98,10 @@ def meet_types(s: Type, t: Type) -> ProperType: return t if isinstance(s, UnionType) and not isinstance(t, UnionType): s, t = t, s + + # Meets/joins require callable type normalization. + s, t = join.normalize_callables(s, t) + return t.accept(TypeMeetVisitor(s)) @@ -97,7 +121,11 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: return original_declared if isinstance(declared, UnionType): return make_simplified_union( - [narrow_declared_type(x, narrowed) for x in declared.relevant_items()] + [ + narrow_declared_type(x, narrowed) + for x in declared.relevant_items() + if is_overlapping_types(x, narrowed, ignore_promotions=True) + ] ) if is_enum_overlapping_union(declared, narrowed): return original_narrowed @@ -127,6 +155,14 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: if declared.type.alt_promote: # Special case: low-level integer type can't be narrowed return original_declared + if ( + isinstance(narrowed, Instance) + and narrowed.type.alt_promote + and narrowed.type.alt_promote is declared.type + ): + # Special case: 'int' can't be narrowed down to a native int type such as + # i64, since they have different runtime representations. + return original_declared return meet_types(original_declared, original_narrowed) elif isinstance(declared, (TupleType, TypeType, LiteralType)): return meet_types(original_declared, original_narrowed) @@ -211,6 +247,7 @@ def is_overlapping_types( right: Type, ignore_promotions: bool = False, prohibit_none_typevar_overlap: bool = False, + ignore_uninhabited: bool = False, ) -> bool: """Can a value of type 'left' also be of type 'right' or vice-versa? @@ -235,6 +272,7 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: right, ignore_promotions=ignore_promotions, prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, + ignore_uninhabited=ignore_uninhabited, ) # We should never encounter this type. @@ -282,8 +320,10 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: ): return True - if is_proper_subtype(left, right, ignore_promotions=ignore_promotions) or is_proper_subtype( - right, left, ignore_promotions=ignore_promotions + if is_proper_subtype( + left, right, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited + ) or is_proper_subtype( + right, left, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited ): return True @@ -425,8 +465,10 @@ def _callable_overlap(left: CallableType, right: CallableType) -> bool: if isinstance(left, Instance) and isinstance(right, Instance): # First we need to handle promotions and structural compatibility for instances # that came as fallbacks, so simply call is_subtype() to avoid code duplication. - if is_subtype(left, right, ignore_promotions=ignore_promotions) or is_subtype( - right, left, ignore_promotions=ignore_promotions + if is_subtype( + left, right, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited + ) or is_subtype( + right, left, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited ): return True @@ -467,7 +509,7 @@ def _callable_overlap(left: CallableType, right: CallableType) -> bool: # Note: it's unclear however, whether returning False is the right thing # to do when inferring reachability -- see https://github.com/python/mypy/issues/5529 - assert type(left) != type(right) + assert type(left) != type(right), f"{type(left)} vs {type(right)}" return False diff --git a/mypy/memprofile.py b/mypy/memprofile.py index 7c479a6480cc..20e18c3c0bf2 100644 --- a/mypy/memprofile.py +++ b/mypy/memprofile.py @@ -35,7 +35,7 @@ def collect_memory_stats() -> tuple[dict[str, int], dict[str, int]]: if hasattr(obj, "__dict__"): # Keep track of which class a particular __dict__ is associated with. inferred[id(obj.__dict__)] = f"{n} (__dict__)" - if isinstance(obj, (Node, Type)): # type: ignore + if isinstance(obj, (Node, Type)): # type: ignore[misc] if hasattr(obj, "__dict__"): for x in obj.__dict__.values(): if isinstance(x, list): diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 4ddf3c9f1a8c..18acb2cd7a71 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -21,13 +21,21 @@ class ErrorMessage(NamedTuple): def format(self, *args: object, **kwargs: object) -> ErrorMessage: return ErrorMessage(self.value.format(*args, **kwargs), code=self.code) + def with_additional_msg(self, info: str) -> ErrorMessage: + return ErrorMessage(self.value + info, code=self.code) + # Invalid types -INVALID_TYPE_RAW_ENUM_VALUE: Final = "Invalid type: try using Literal[{}.{}] instead?" +INVALID_TYPE_RAW_ENUM_VALUE: Final = ErrorMessage( + "Invalid type: try using Literal[{}.{}] instead?", codes.VALID_TYPE +) # Type checker error message constants NO_RETURN_VALUE_EXPECTED: Final = ErrorMessage("No return value expected", codes.RETURN_VALUE) MISSING_RETURN_STATEMENT: Final = ErrorMessage("Missing return statement", codes.RETURN) +EMPTY_BODY_ABSTRACT: Final = ErrorMessage( + "If the method is meant to be abstract, use @abc.abstractmethod", codes.EMPTY_BODY +) INVALID_IMPLICIT_RETURN: Final = ErrorMessage("Implicit return in function which does not return") INCOMPATIBLE_RETURN_VALUE_TYPE: Final = ErrorMessage( "Incompatible return value type", codes.RETURN_VALUE @@ -36,6 +44,9 @@ def format(self, *args: object, **kwargs: object) -> ErrorMessage: NO_RETURN_EXPECTED: Final = ErrorMessage("Return statement in function which does not return") INVALID_EXCEPTION: Final = ErrorMessage("Exception must be derived from BaseException") INVALID_EXCEPTION_TYPE: Final = ErrorMessage("Exception type must be derived from BaseException") +INVALID_EXCEPTION_GROUP: Final = ErrorMessage( + "Exception type in except* cannot derive from BaseExceptionGroup" +) RETURN_IN_ASYNC_GENERATOR: Final = ErrorMessage( '"return" with value in async generator is not allowed' ) @@ -47,8 +58,10 @@ def format(self, *args: object, **kwargs: object) -> ErrorMessage: "supertypes" ) YIELD_VALUE_EXPECTED: Final = ErrorMessage("Yield value expected") -INCOMPATIBLE_TYPES: Final = "Incompatible types" -INCOMPATIBLE_TYPES_IN_ASSIGNMENT: Final = "Incompatible types in assignment" +INCOMPATIBLE_TYPES: Final = ErrorMessage("Incompatible types") +INCOMPATIBLE_TYPES_IN_ASSIGNMENT: Final = ErrorMessage( + "Incompatible types in assignment", code=codes.ASSIGNMENT +) INCOMPATIBLE_TYPES_IN_AWAIT: Final = ErrorMessage('Incompatible types in "await"') INCOMPATIBLE_REDEFINITION: Final = ErrorMessage("Incompatible redefinition") INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AENTER: Final = ( @@ -94,7 +107,7 @@ def format(self, *args: object, **kwargs: object) -> ErrorMessage: FUNCTION_PARAMETER_CANNOT_BE_COVARIANT: Final = ErrorMessage( "Cannot use a covariant type variable as a parameter" ) -INCOMPATIBLE_IMPORT_OF: Final = "Incompatible import of" +INCOMPATIBLE_IMPORT_OF: Final = ErrorMessage('Incompatible import of "{}"', code=codes.ASSIGNMENT) FUNCTION_TYPE_EXPECTED: Final = ErrorMessage( "Function is missing a type annotation", codes.NO_UNTYPED_DEF ) @@ -138,7 +151,7 @@ def format(self, *args: object, **kwargs: object) -> ErrorMessage: code=codes.TRUTHY_BOOL, ) FUNCTION_ALWAYS_TRUE: Final = ErrorMessage( - "Function {} could always be true in boolean context", code=codes.TRUTHY_BOOL + "Function {} could always be true in boolean context", code=codes.TRUTHY_FUNCTION ) NOT_CALLABLE: Final = "{} not callable" TYPE_MUST_BE_USED: Final = "Value of type {} must be used" @@ -165,7 +178,7 @@ def format(self, *args: object, **kwargs: object) -> ErrorMessage: TYPEVAR_UNEXPECTED_ARGUMENT: Final = 'Unexpected argument to "TypeVar()"' UNBOUND_TYPEVAR: Final = ( "A function returning TypeVar should receive at least " - "one argument containing the same Typevar" + "one argument containing the same TypeVar" ) # Super diff --git a/mypy/messages.py b/mypy/messages.py index d93541e94c9c..4e762faa0b32 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -54,6 +54,7 @@ IS_CLASS_OR_STATIC, IS_CLASSVAR, IS_SETTABLE, + IS_VAR, find_member, get_member_flags, is_same_type, @@ -79,10 +80,12 @@ TypedDictType, TypeOfAny, TypeType, + TypeVarTupleType, TypeVarType, UnboundType, UninhabitedType, UnionType, + UnpackType, get_proper_type, get_proper_types, ) @@ -162,7 +165,10 @@ def __init__(self, errors: Errors, modules: dict[str, MypyFile]) -> None: # def filter_errors( - self, *, filter_errors: bool = True, save_filtered_errors: bool = False + self, + *, + filter_errors: bool | Callable[[str, ErrorInfo], bool] = True, + save_filtered_errors: bool = False, ) -> ErrorWatcher: return ErrorWatcher( self.errors, filter_errors=filter_errors, save_filtered_errors=save_filtered_errors @@ -403,32 +409,43 @@ def has_no_attr( if not self.are_type_names_disabled(): failed = False if isinstance(original_type, Instance) and original_type.type.names: - alternatives = set(original_type.type.names.keys()) - - if module_symbol_table is not None: - alternatives |= {key for key in module_symbol_table.keys()} - - # in some situations, the member is in the alternatives set - # but since we're in this function, we shouldn't suggest it - if member in alternatives: - alternatives.remove(member) - - matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] - matches.extend(best_matches(member, alternatives)[:3]) - if member == "__aiter__" and matches == ["__iter__"]: - matches = [] # Avoid misleading suggestion - if matches: + if ( + module_symbol_table is not None + and member in module_symbol_table + and not module_symbol_table[member].module_public + ): self.fail( - '{} has no attribute "{}"; maybe {}?{}'.format( - format_type(original_type), - member, - pretty_seq(matches, "or"), - extra, - ), + f"{format_type(original_type, module_names=True)} does not " + f'explicitly export attribute "{member}"', context, code=codes.ATTR_DEFINED, ) failed = True + else: + alternatives = set(original_type.type.names.keys()) + if module_symbol_table is not None: + alternatives |= { + k for k, v in module_symbol_table.items() if v.module_public + } + # Rare but possible, see e.g. testNewAnalyzerCyclicDefinitionCrossModule + alternatives.discard(member) + + matches = [m for m in COMMON_MISTAKES.get(member, []) if m in alternatives] + matches.extend(best_matches(member, alternatives)[:3]) + if member == "__aiter__" and matches == ["__iter__"]: + matches = [] # Avoid misleading suggestion + if matches: + self.fail( + '{} has no attribute "{}"; maybe {}?{}'.format( + format_type(original_type), + member, + pretty_seq(matches, "or"), + extra, + ), + context, + code=codes.ATTR_DEFINED, + ) + failed = True if not failed: self.fail( '{} has no attribute "{}"{}'.format( @@ -578,20 +595,18 @@ def incompatible_argument( ) return codes.INDEX else: - msg = "{} (expression has type {}, target has type {})" arg_type_str, callee_type_str = format_type_distinctly( arg_type, callee.arg_types[n - 1] ) - self.fail( - msg.format( - message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, - arg_type_str, - callee_type_str, - ), - context, - code=codes.ASSIGNMENT, + info = ( + f" (expression has type {arg_type_str}, " + f"target has type {callee_type_str})" + ) + error_msg = ( + message_registry.INCOMPATIBLE_TYPES_IN_ASSIGNMENT.with_additional_msg(info) ) - return codes.ASSIGNMENT + self.fail(error_msg.value, context, code=error_msg.code) + return error_msg.code target = f"to {name} " @@ -740,7 +755,9 @@ def incompatible_argument_note( context: Context, code: ErrorCode | None, ) -> None: - if isinstance(original_caller_type, (Instance, TupleType, TypedDictType)): + if isinstance( + original_caller_type, (Instance, TupleType, TypedDictType, TypeType, CallableType) + ): if isinstance(callee_type, Instance) and callee_type.type.is_protocol: self.report_protocol_problems( original_caller_type, callee_type, context, code=code @@ -787,8 +804,8 @@ def maybe_note_concatenate_pos_args( if names: missing_arguments = '"' + '", "'.join(names) + '"' self.note( - f'This may be because "{original_caller_type.name}" has arguments ' - f"named: {missing_arguments}", + f'This is likely because "{original_caller_type.name}" has named arguments: ' + f"{missing_arguments}. Consider marking them positional-only", context, code=code, ) @@ -1210,6 +1227,9 @@ def invalid_keyword_var_arg(self, typ: Type, is_mapping: bool, context: Context) def undefined_in_superclass(self, member: str, context: Context) -> None: self.fail(f'"{member}" undefined in superclass', context) + def variable_may_be_undefined(self, name: str, context: Context) -> None: + self.fail(f'Name "{name}" may be undefined', context, code=codes.PARTIALLY_DEFINED) + def first_argument_for_super_must_be_type(self, actual: Type, context: Context) -> None: actual = get_proper_type(actual) if isinstance(actual, Instance): @@ -1224,6 +1244,14 @@ def first_argument_for_super_must_be_type(self, actual: Type, context: Context) code=codes.ARG_TYPE, ) + def unsafe_super(self, method: str, cls: str, ctx: Context) -> None: + self.fail( + 'Call to abstract method "{}" of "{}" with trivial body' + " via super() is unsafe".format(method, cls), + ctx, + code=codes.SAFE_SUPER, + ) + def too_few_string_formatting_arguments(self, context: Context) -> None: self.fail("Not enough arguments for format string", context, code=codes.STRING_FORMATTING) @@ -1295,8 +1323,17 @@ def incompatible_self_argument( context, ) - def incompatible_conditional_function_def(self, defn: FuncDef) -> None: - self.fail("All conditional function variants must have identical " "signatures", defn) + def incompatible_conditional_function_def( + self, defn: FuncDef, old_type: FunctionLike, new_type: FunctionLike + ) -> None: + self.fail("All conditional function variants must have identical signatures", defn) + if isinstance(old_type, (CallableType, Overloaded)) and isinstance( + new_type, (CallableType, Overloaded) + ): + self.note("Original:", defn) + self.pretty_callable_or_overload(old_type, defn, offset=4) + self.note("Redefinition:", defn) + self.pretty_callable_or_overload(new_type, defn, offset=4) def cannot_instantiate_abstract_class( self, class_name: str, abstract_attributes: dict[str, bool], context: Context @@ -1317,15 +1354,16 @@ def cannot_instantiate_abstract_class( return if len(attrs_with_none) == 1: note = ( - "The following method was marked implicitly abstract because it has an empty " - "function body: {}. If it is not meant to be abstract, explicitly return None." + f"{attrs_with_none[0]} is implicitly abstract because it has an empty function " + "body. If it is not meant to be abstract, explicitly `return` or `return None`." ) else: note = ( "The following methods were marked implicitly abstract because they have empty " - "function bodies: {}. If they are not meant to be abstract, explicitly return None." + f"function bodies: {format_string_list(attrs_with_none)}. " + "If they are not meant to be abstract, explicitly `return` or `return None`." ) - self.note(note.format(format_string_list(attrs_with_none)), context, code=codes.ABSTRACT) + self.note(note, context, code=codes.ABSTRACT) def base_class_definitions_incompatible( self, name: str, base1: TypeInfo, base2: TypeInfo, context: Context @@ -1520,23 +1558,30 @@ def need_annotation_for_var( ) -> None: hint = "" has_variable_annotations = not python_version or python_version >= (3, 6) + pep604_supported = not python_version or python_version >= (3, 10) + # type to recommend the user adds + recommended_type = None # Only gives hint if it's a variable declaration and the partial type is a builtin type - if ( - python_version - and isinstance(node, Var) - and isinstance(node.type, PartialType) - and node.type.type - and node.type.type.fullname in reverse_builtin_aliases - ): - alias = reverse_builtin_aliases[node.type.type.fullname] - alias = alias.split(".")[-1] + if python_version and isinstance(node, Var) and isinstance(node.type, PartialType): type_dec = "" - if alias == "Dict": - type_dec = f"{type_dec}, {type_dec}" + if not node.type.type: + # partial None + if pep604_supported: + recommended_type = f"{type_dec} | None" + else: + recommended_type = f"Optional[{type_dec}]" + elif node.type.type.fullname in reverse_builtin_aliases: + # partial types other than partial None + alias = reverse_builtin_aliases[node.type.type.fullname] + alias = alias.split(".")[-1] + if alias == "Dict": + type_dec = f"{type_dec}, {type_dec}" + recommended_type = f"{alias}[{type_dec}]" + if recommended_type is not None: if has_variable_annotations: - hint = f' (hint: "{node.name}: {alias}[{type_dec}] = ...")' + hint = f' (hint: "{node.name}: {recommended_type} = ...")' else: - hint = f' (hint: "{node.name} = ... # type: {alias}[{type_dec}]")' + hint = f' (hint: "{node.name} = ... # type: {recommended_type}")' if has_variable_annotations: needed = "annotation" @@ -1716,7 +1761,9 @@ def concrete_only_assign(self, typ: Type, context: Context) -> None: def concrete_only_call(self, typ: Type, context: Context) -> None: self.fail( - f"Only concrete class can be given where {format_type(typ)} is expected", context + f"Only concrete class can be given where {format_type(typ)} is expected", + context, + code=codes.TYPE_ABSTRACT, ) def cannot_use_function_with_type( @@ -1788,7 +1835,7 @@ def impossible_intersection( def report_protocol_problems( self, - subtype: Instance | TupleType | TypedDictType, + subtype: Instance | TupleType | TypedDictType | TypeType | CallableType, supertype: Instance, context: Context, *, @@ -1808,15 +1855,16 @@ def report_protocol_problems( exclusions: dict[type, list[str]] = { TypedDictType: ["typing.Mapping"], TupleType: ["typing.Iterable", "typing.Sequence"], - Instance: [], } - if supertype.type.fullname in exclusions[type(subtype)]: + if supertype.type.fullname in exclusions.get(type(subtype), []): return if any(isinstance(tp, UninhabitedType) for tp in get_proper_types(supertype.args)): # We don't want to add notes for failed inference (e.g. Iterable[]). # This will be only confusing a user even more. return + class_obj = False + is_module = False if isinstance(subtype, TupleType): if not isinstance(subtype.partial_fallback, Instance): return @@ -1825,6 +1873,23 @@ def report_protocol_problems( if not isinstance(subtype.fallback, Instance): return subtype = subtype.fallback + elif isinstance(subtype, TypeType): + if not isinstance(subtype.item, Instance): + return + class_obj = True + subtype = subtype.item + elif isinstance(subtype, CallableType): + if not subtype.is_type_obj(): + return + ret_type = get_proper_type(subtype.ret_type) + if isinstance(ret_type, TupleType): + ret_type = ret_type.partial_fallback + if not isinstance(ret_type, Instance): + return + class_obj = True + subtype = ret_type + if subtype.extra_attrs and subtype.extra_attrs.mod_name: + is_module = True # Report missing members missing = get_missing_protocol_members(subtype, supertype) @@ -1833,30 +1898,36 @@ def report_protocol_problems( and len(missing) < len(supertype.type.protocol_members) and len(missing) <= MAX_ITEMS ): - self.note( - '"{}" is missing following "{}" protocol member{}:'.format( - subtype.type.name, supertype.type.name, plural_s(missing) - ), - context, - code=code, - ) - self.note(", ".join(missing), context, offset=OFFSET, code=code) + if missing == ["__call__"] and class_obj: + self.note( + '"{}" has constructor incompatible with "__call__" of "{}"'.format( + subtype.type.name, supertype.type.name + ), + context, + code=code, + ) + else: + self.note( + '"{}" is missing following "{}" protocol member{}:'.format( + subtype.type.name, supertype.type.name, plural_s(missing) + ), + context, + code=code, + ) + self.note(", ".join(missing), context, offset=OFFSET, code=code) elif len(missing) > MAX_ITEMS or len(missing) == len(supertype.type.protocol_members): # This is an obviously wrong type: too many missing members return # Report member type conflicts - conflict_types = get_conflict_protocol_types(subtype, supertype) + conflict_types = get_conflict_protocol_types(subtype, supertype, class_obj=class_obj) if conflict_types and ( not is_subtype(subtype, erase_type(supertype)) or not subtype.type.defn.type_vars or not supertype.type.defn.type_vars ): - self.note( - f"Following member(s) of {format_type(subtype)} have conflicts:", - context, - code=code, - ) + type_name = format_type(subtype, module_names=True) + self.note(f"Following member(s) of {type_name} have conflicts:", context, code=code) for name, got, exp in conflict_types[:MAX_ITEMS]: exp = get_proper_type(exp) got = get_proper_type(got) @@ -1872,29 +1943,43 @@ def report_protocol_problems( else: self.note("Expected:", context, offset=OFFSET, code=code) if isinstance(exp, CallableType): - self.note(pretty_callable(exp), context, offset=2 * OFFSET, code=code) + self.note( + pretty_callable(exp, skip_self=class_obj or is_module), + context, + offset=2 * OFFSET, + code=code, + ) else: assert isinstance(exp, Overloaded) - self.pretty_overload(exp, context, 2 * OFFSET, code=code) + self.pretty_overload( + exp, context, 2 * OFFSET, code=code, skip_self=class_obj or is_module + ) self.note("Got:", context, offset=OFFSET, code=code) if isinstance(got, CallableType): - self.note(pretty_callable(got), context, offset=2 * OFFSET, code=code) + self.note( + pretty_callable(got, skip_self=class_obj or is_module), + context, + offset=2 * OFFSET, + code=code, + ) else: assert isinstance(got, Overloaded) - self.pretty_overload(got, context, 2 * OFFSET, code=code) + self.pretty_overload( + got, context, 2 * OFFSET, code=code, skip_self=class_obj or is_module + ) self.print_more(conflict_types, context, OFFSET, MAX_ITEMS, code=code) # Report flag conflicts (i.e. settable vs read-only etc.) - conflict_flags = get_bad_protocol_flags(subtype, supertype) + conflict_flags = get_bad_protocol_flags(subtype, supertype, class_obj=class_obj) for name, subflags, superflags in conflict_flags[:MAX_ITEMS]: - if IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: + if not class_obj and IS_CLASSVAR in subflags and IS_CLASSVAR not in superflags: self.note( "Protocol member {}.{} expected instance variable," " got class variable".format(supertype.type.name, name), context, code=code, ) - if IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: + if not class_obj and IS_CLASSVAR in superflags and IS_CLASSVAR not in subflags: self.note( "Protocol member {}.{} expected class variable," " got instance variable".format(supertype.type.name, name), @@ -1916,6 +2001,25 @@ def report_protocol_problems( context, code=code, ) + if ( + class_obj + and IS_VAR in superflags + and (IS_VAR in subflags and IS_CLASSVAR not in subflags) + ): + self.note( + "Only class variables allowed for class object access on protocols," + ' {} is an instance variable of "{}"'.format(name, subtype.type.name), + context, + code=code, + ) + if class_obj and IS_CLASSVAR in superflags: + self.note( + "ClassVar protocol member {}.{} can never be matched by a class object".format( + supertype.type.name, name + ), + context, + code=code, + ) self.print_more(conflict_flags, context, OFFSET, MAX_ITEMS, code=code) def pretty_overload( @@ -1927,6 +2031,7 @@ def pretty_overload( add_class_or_static_decorator: bool = False, allow_dups: bool = False, code: ErrorCode | None = None, + skip_self: bool = False, ) -> None: for item in tp.items: self.note("@overload", context, offset=offset, allow_dups=allow_dups, code=code) @@ -1937,7 +2042,11 @@ def pretty_overload( self.note(decorator, context, offset=offset, allow_dups=allow_dups, code=code) self.note( - pretty_callable(item), context, offset=offset, allow_dups=allow_dups, code=code + pretty_callable(item, skip_self=skip_self), + context, + offset=offset, + allow_dups=allow_dups, + code=code, ) def print_more( @@ -1962,10 +2071,9 @@ def try_report_long_tuple_assignment_error( subtype: ProperType, supertype: ProperType, context: Context, - msg: str = message_registry.INCOMPATIBLE_TYPES, + msg: message_registry.ErrorMessage, subtype_label: str | None = None, supertype_label: str | None = None, - code: ErrorCode | None = None, ) -> bool: """Try to generate meaningful error message for very long tuple assignment @@ -1980,26 +2088,25 @@ def try_report_long_tuple_assignment_error( ): lhs_type = supertype.args[0] lhs_types = [lhs_type] * len(subtype.items) - self.generate_incompatible_tuple_error( - lhs_types, subtype.items, context, msg, code - ) + self.generate_incompatible_tuple_error(lhs_types, subtype.items, context, msg) return True elif isinstance(supertype, TupleType) and ( len(subtype.items) > 10 or len(supertype.items) > 10 ): if len(subtype.items) != len(supertype.items): if supertype_label is not None and subtype_label is not None: - error_msg = "{} ({} {}, {} {})".format( - msg, - subtype_label, - self.format_long_tuple_type(subtype), - supertype_label, - self.format_long_tuple_type(supertype), + msg = msg.with_additional_msg( + " ({} {}, {} {})".format( + subtype_label, + self.format_long_tuple_type(subtype), + supertype_label, + self.format_long_tuple_type(supertype), + ) ) - self.fail(error_msg, context, code=code) + self.fail(msg.value, context, code=msg.code) return True self.generate_incompatible_tuple_error( - supertype.items, subtype.items, context, msg, code + supertype.items, subtype.items, context, msg ) return True return False @@ -2019,8 +2126,7 @@ def generate_incompatible_tuple_error( lhs_types: list[Type], rhs_types: list[Type], context: Context, - msg: str = message_registry.INCOMPATIBLE_TYPES, - code: ErrorCode | None = None, + msg: message_registry.ErrorMessage, ) -> None: """Generate error message for individual incompatible tuple pairs""" error_cnt = 0 @@ -2035,14 +2141,15 @@ def generate_incompatible_tuple_error( ) error_cnt += 1 - error_msg = msg + f" ({str(error_cnt)} tuple items are incompatible" + info = f" ({str(error_cnt)} tuple items are incompatible" if error_cnt - 3 > 0: - error_msg += f"; {str(error_cnt - 3)} items are omitted)" + info += f"; {str(error_cnt - 3)} items are omitted)" else: - error_msg += ")" - self.fail(error_msg, context, code=code) + info += ")" + msg = msg.with_additional_msg(info) + self.fail(msg.value, context, code=msg.code) for note in notes: - self.note(note, context, code=code) + self.note(note, context, code=msg.code) def add_fixture_note(self, fullname: str, ctx: Context) -> None: self.note(f'Maybe your test fixture does not define "{fullname}"?', ctx) @@ -2054,6 +2161,14 @@ def add_fixture_note(self, fullname: str, ctx: Context) -> None: ctx, ) + def annotation_in_unchecked_function(self, context: Context) -> None: + self.note( + "By default the bodies of untyped functions are not checked," + " consider using --check-untyped-defs", + context, + code=codes.ANNOTATION_UNCHECKED, + ) + def quote_type_string(type_string: str) -> str: """Quotes a type representation for use in messages.""" @@ -2092,7 +2207,9 @@ def format_callable_args( return ", ".join(arg_strings) -def format_type_inner(typ: Type, verbosity: int, fullnames: set[str] | None) -> str: +def format_type_inner( + typ: Type, verbosity: int, fullnames: set[str] | None, module_names: bool = False +) -> str: """ Convert a type to a relatively short string suitable for error messages. @@ -2132,7 +2249,10 @@ def format_literal_value(typ: LiteralType) -> str: # Get the short name of the type. if itype.type.fullname in ("types.ModuleType", "_importlib_modulespec.ModuleType"): # Make some common error messages simpler and tidier. - return "Module" + base_str = "Module" + if itype.extra_attrs and itype.extra_attrs.mod_name and module_names: + return f"{base_str} {itype.extra_attrs.mod_name}" + return base_str if verbosity >= 2 or (fullnames and itype.type.fullname in fullnames): base_str = itype.type.fullname else: @@ -2150,9 +2270,14 @@ def format_literal_value(typ: LiteralType) -> str: else: # There are type arguments. Convert the arguments to strings. return f"{base_str}[{format_list(itype.args)}]" + elif isinstance(typ, UnpackType): + return f"Unpack[{format(typ.type)}]" elif isinstance(typ, TypeVarType): # This is similar to non-generic instance types. return typ.name + elif isinstance(typ, TypeVarTupleType): + # This is similar to non-generic instance types. + return typ.name elif isinstance(typ, ParamSpecType): # Concatenate[..., P] if typ.prefix.arg_types: @@ -2306,7 +2431,7 @@ def find_type_overlaps(*types: Type) -> set[str]: return overlaps -def format_type(typ: Type, verbosity: int = 0) -> str: +def format_type(typ: Type, verbosity: int = 0, module_names: bool = False) -> str: """ Convert a type to a relatively short string suitable for error messages. @@ -2317,10 +2442,10 @@ def format_type(typ: Type, verbosity: int = 0) -> str: modification of the formatted string is required, callers should use format_type_bare. """ - return quote_type_string(format_type_bare(typ, verbosity)) + return quote_type_string(format_type_bare(typ, verbosity, module_names)) -def format_type_bare(typ: Type, verbosity: int = 0) -> str: +def format_type_bare(typ: Type, verbosity: int = 0, module_names: bool = False) -> str: """ Convert a type to a relatively short string suitable for error messages. @@ -2332,7 +2457,7 @@ def format_type_bare(typ: Type, verbosity: int = 0) -> str: instead. (The caller may want to use quote_type_string after processing has happened, to maintain consistent quoting in messages.) """ - return format_type_inner(typ, verbosity, find_type_overlaps(typ)) + return format_type_inner(typ, verbosity, find_type_overlaps(typ), module_names) def format_type_distinctly(*types: Type, bare: bool = False) -> tuple[str, ...]: @@ -2370,13 +2495,18 @@ def pretty_class_or_static_decorator(tp: CallableType) -> str | None: return None -def pretty_callable(tp: CallableType) -> str: +def pretty_callable(tp: CallableType, skip_self: bool = False) -> str: """Return a nice easily-readable representation of a callable type. For example: def [T <: int] f(self, x: int, y: T) -> None + + If skip_self is True, print an actual callable type, as it would appear + when bound on an instance/class, rather than how it would appear in the + defining statement. """ s = "" asterisk = False + slash = False for i in range(len(tp.arg_types)): if s: s += ", " @@ -2391,26 +2521,35 @@ def [T <: int] f(self, x: int, y: T) -> None name = tp.arg_names[i] if name: s += name + ": " - s += format_type_bare(tp.arg_types[i]) + type_str = format_type_bare(tp.arg_types[i]) + if tp.arg_kinds[i] == ARG_STAR2 and tp.unpack_kwargs: + type_str = f"Unpack[{type_str}]" + s += type_str if tp.arg_kinds[i].is_optional(): s += " = ..." + if ( + not slash + and tp.arg_kinds[i].is_positional() + and name is None + and ( + i == len(tp.arg_types) - 1 + or (tp.arg_names[i + 1] is not None or not tp.arg_kinds[i + 1].is_positional()) + ) + ): + s += ", /" + slash = True # If we got a "special arg" (i.e: self, cls, etc...), prepend it to the arg list - if ( - isinstance(tp.definition, FuncDef) - and tp.definition.name is not None - and hasattr(tp.definition, "arguments") - ): - definition_args = [arg.variable.name for arg in tp.definition.arguments] + if isinstance(tp.definition, FuncDef) and hasattr(tp.definition, "arguments"): + definition_arg_names = [arg.variable.name for arg in tp.definition.arguments] if ( - definition_args - and tp.arg_names != definition_args - and len(definition_args) > 0 - and definition_args[0] + len(definition_arg_names) > len(tp.arg_names) + and definition_arg_names[0] + and not skip_self ): if s: s = ", " + s - s = definition_args[0] + s + s = definition_arg_names[0] + s s = f"{tp.definition.name}({s})" elif tp.name: first_arg = tp.def_extras.get("first_arg") @@ -2474,7 +2613,9 @@ def get_missing_protocol_members(left: Instance, right: Instance) -> list[str]: return missing -def get_conflict_protocol_types(left: Instance, right: Instance) -> list[tuple[str, Type, Type]]: +def get_conflict_protocol_types( + left: Instance, right: Instance, class_obj: bool = False +) -> list[tuple[str, Type, Type]]: """Find members that are defined in 'left' but have incompatible types. Return them as a list of ('member', 'got', 'expected'). """ @@ -2485,11 +2626,11 @@ def get_conflict_protocol_types(left: Instance, right: Instance) -> list[tuple[s continue supertype = find_member(member, right, left) assert supertype is not None - subtype = find_member(member, left, left) + subtype = find_member(member, left, left, class_obj=class_obj) if not subtype: continue is_compat = is_subtype(subtype, supertype, ignore_pos_arg_names=True) - if IS_SETTABLE in get_member_flags(member, right.type): + if IS_SETTABLE in get_member_flags(member, right): is_compat = is_compat and is_subtype(supertype, subtype) if not is_compat: conflicts.append((member, subtype, supertype)) @@ -2497,7 +2638,7 @@ def get_conflict_protocol_types(left: Instance, right: Instance) -> list[tuple[s def get_bad_protocol_flags( - left: Instance, right: Instance + left: Instance, right: Instance, class_obj: bool = False ) -> list[tuple[str, set[int], set[int]]]: """Return all incompatible attribute flags for members that are present in both 'left' and 'right'. @@ -2506,11 +2647,7 @@ def get_bad_protocol_flags( all_flags: list[tuple[str, set[int], set[int]]] = [] for member in right.type.protocol_members: if find_member(member, left, left): - item = ( - member, - get_member_flags(member, left.type), - get_member_flags(member, right.type), - ) + item = (member, get_member_flags(member, left), get_member_flags(member, right)) all_flags.append(item) bad_flags = [] for name, subflags, superflags in all_flags: @@ -2523,6 +2660,11 @@ def get_bad_protocol_flags( and IS_SETTABLE not in subflags or IS_CLASS_OR_STATIC in superflags and IS_CLASS_OR_STATIC not in subflags + or class_obj + and IS_VAR in superflags + and IS_CLASSVAR not in subflags + or class_obj + and IS_CLASSVAR in superflags ): bad_flags.append((name, subflags, superflags)) return bad_flags @@ -2588,11 +2730,22 @@ def for_function(callee: CallableType) -> str: return "" +def wrong_type_arg_count(n: int, act: str, name: str) -> str: + s = f"{n} type arguments" + if n == 0: + s = "no type arguments" + elif n == 1: + s = "1 type argument" + if act == "0": + act = "none" + return f'"{name}" expects {s}, but {act} given' + + def find_defining_module(modules: dict[str, MypyFile], typ: CallableType) -> MypyFile | None: if not typ.definition: return None fullname = typ.definition.fullname - if fullname is not None and "." in fullname: + if "." in fullname: for i in range(fullname.count(".")): module_name = fullname.rsplit(".", i + 1)[0] try: diff --git a/mypy/metastore.py b/mypy/metastore.py index 8a8a3088ca76..16cbd5adc9c8 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -185,10 +185,14 @@ def _query(self, name: str, field: str) -> Any: return results[0][0] def getmtime(self, name: str) -> float: - return self._query(name, "mtime") + mtime = self._query(name, "mtime") + assert isinstance(mtime, float) + return mtime def read(self, name: str) -> str: - return self._query(name, "data") + data = self._query(name, "data") + assert isinstance(data, str) + return data def write(self, name: str, data: str, mtime: float | None = None) -> bool: import sqlite3 diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index aaa8216ae435..5d542b154906 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -28,7 +28,7 @@ from mypy.fscache import FileSystemCache from mypy.nodes import MypyFile from mypy.options import Options -from mypy.stubinfo import is_legacy_bundled_package +from mypy.stubinfo import approved_stub_package_exists # Paths to be searched in find_module(). @@ -89,9 +89,7 @@ def error_message_templates(self, daemon: bool) -> tuple[str, list[str]]: ) notes = [doc_link] elif self is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: - msg = ( - 'Library stubs not installed for "{module}" (or incompatible with Python {pyver})' - ) + msg = 'Library stubs not installed for "{module}"' notes = ['Hint: "python3 -m pip install {stub_dist}"'] if not daemon: notes.append( @@ -338,13 +336,13 @@ def _find_module_non_stub_helper( # If this is not a directory then we can't traverse further into it if not self.fscache.isdir(dir_path): break - if is_legacy_bundled_package(components[0]): + if approved_stub_package_exists(components[0]): if len(components) == 1 or ( self.find_module(components[0]) is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED ): return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED - if is_legacy_bundled_package(".".join(components[:2])): + if approved_stub_package_exists(".".join(components[:2])): return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED if plausible_match: return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS diff --git a/mypy/nodes.py b/mypy/nodes.py index 765feb171b9b..740fe288723a 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -10,7 +10,6 @@ TYPE_CHECKING, Any, Callable, - DefaultDict, Dict, Iterator, Optional, @@ -306,7 +305,7 @@ class MypyFile(SymbolNode): # Top-level definitions and statements defs: list[Statement] # Type alias dependencies as mapping from target to set of alias full names - alias_deps: DefaultDict[str, set[str]] + alias_deps: defaultdict[str, set[str]] # Is there a UTF-8 BOM at the start? is_bom: bool names: SymbolTable @@ -759,7 +758,12 @@ def is_dynamic(self) -> bool: return self.type is None -FUNCDEF_FLAGS: Final = FUNCITEM_FLAGS + ["is_decorated", "is_conditional"] +FUNCDEF_FLAGS: Final = FUNCITEM_FLAGS + [ + "is_decorated", + "is_conditional", + "is_trivial_body", + "is_mypy_only", +] # Abstract status of a function NOT_ABSTRACT: Final = 0 @@ -782,6 +786,8 @@ class FuncDef(FuncItem, SymbolNode, Statement): "abstract_status", "original_def", "deco_line", + "is_trivial_body", + "is_mypy_only", ) # Note that all __init__ args must have default values @@ -797,11 +803,16 @@ def __init__( self.is_decorated = False self.is_conditional = False # Defined conditionally (within block)? self.abstract_status = NOT_ABSTRACT + # Is this an abstract method with trivial body? + # Such methods can't be called via super(). + self.is_trivial_body = False self.is_final = False # Original conditional definition self.original_def: None | FuncDef | Var | Decorator = None - # Used for error reporting (to keep backwad compatibility with pre-3.8) + # Used for error reporting (to keep backward compatibility with pre-3.8) self.deco_line: int | None = None + # Definitions that appear in if TYPE_CHECKING are marked with this flag. + self.is_mypy_only = False @property def name(self) -> str: @@ -940,6 +951,7 @@ def deserialize(cls, data: JsonDict) -> Decorator: "explicit_self_type", "is_ready", "is_inferred", + "invalid_partial_type", "from_module_getattr", "has_explicit_value", "allow_incompatible_override", @@ -976,6 +988,7 @@ class Var(SymbolNode): "from_module_getattr", "has_explicit_value", "allow_incompatible_override", + "invalid_partial_type", ) def __init__(self, name: str, type: mypy.types.Type | None = None) -> None: @@ -1025,6 +1038,9 @@ def __init__(self, name: str, type: mypy.types.Type | None = None) -> None: self.has_explicit_value = False # If True, subclasses can override this with an incompatible type. self.allow_incompatible_override = False + # If True, this means we didn't manage to infer full type and fall back to + # something like list[Any]. We may decide to not use such types as context. + self.invalid_partial_type = False @property def name(self) -> str: @@ -1109,7 +1125,7 @@ def __init__( ) -> None: super().__init__() self.name = name - self.fullname = None # type: ignore + self.fullname = None # type: ignore[assignment] self.defs = defs self.type_vars = type_vars or [] self.base_type_exprs = base_type_exprs or [] @@ -1469,7 +1485,7 @@ def accept(self, visitor: StatementVisitor[T]) -> T: class TryStmt(Statement): - __slots__ = ("body", "types", "vars", "handlers", "else_body", "finally_body") + __slots__ = ("body", "types", "vars", "handlers", "else_body", "finally_body", "is_star") body: Block # Try body # Plain 'except:' also possible @@ -1478,6 +1494,8 @@ class TryStmt(Statement): handlers: list[Block] # Except bodies else_body: Block | None finally_body: Block | None + # Whether this is try ... except* (added in Python 3.11) + is_star: bool def __init__( self, @@ -1495,6 +1513,7 @@ def __init__( self.handlers = handlers self.else_body = else_body self.finally_body = finally_body + self.is_star = False def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_try_stmt(self) @@ -2526,9 +2545,9 @@ class PromoteExpr(Expression): __slots__ = ("type",) - type: mypy.types.Type + type: mypy.types.ProperType - def __init__(self, type: mypy.types.Type) -> None: + def __init__(self, type: mypy.types.ProperType) -> None: super().__init__() self.type = type @@ -2753,7 +2772,7 @@ class is generic then it will be a type constructor of higher kind. # even though it's not a subclass in Python. The non-standard # `@_promote` decorator introduces this, and there are also # several builtin examples, in particular `int` -> `float`. - _promote: list[mypy.types.Type] + _promote: list[mypy.types.ProperType] # This is used for promoting native integer types such as 'i64' to # 'int'. (_promote is used for the other direction.) This only @@ -2854,6 +2873,7 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None self.metadata = {} def add_type_vars(self) -> None: + self.has_type_var_tuple_type = False if self.defn.type_vars: for i, vd in enumerate(self.defn.type_vars): if isinstance(vd, mypy.types.ParamSpecType): @@ -2902,7 +2922,10 @@ def protocol_members(self) -> list[str]: assert self.mro, "This property can be only accessed after MRO is (re-)calculated" for base in self.mro[:-1]: # we skip "object" since everyone implements it if base.is_protocol: - for name in base.names: + for name, node in base.names.items(): + if isinstance(node.node, (TypeAlias, TypeVarExpr)): + # These are auxiliary definitions (and type aliases are prohibited). + continue members.add(name) return sorted(list(members)) @@ -3080,7 +3103,12 @@ def deserialize(cls, data: JsonDict) -> TypeInfo: ti.type_vars = data["type_vars"] ti.has_param_spec_type = data["has_param_spec_type"] ti.bases = [mypy.types.Instance.deserialize(b) for b in data["bases"]] - ti._promote = [mypy.types.deserialize_type(p) for p in data["_promote"]] + _promote = [] + for p in data["_promote"]: + t = mypy.types.deserialize_type(p) + assert isinstance(t, mypy.types.ProperType) + _promote.append(t) + ti._promote = _promote ti.declared_metaclass = ( None if data["declared_metaclass"] is None @@ -3146,10 +3174,10 @@ class FakeInfo(TypeInfo): def __init__(self, msg: str) -> None: self.msg = msg - def __getattribute__(self, attr: str) -> None: + def __getattribute__(self, attr: str) -> type: # Handle __class__ so that isinstance still works... if attr == "__class__": - return object.__getattribute__(self, attr) + return object.__getattribute__(self, attr) # type: ignore[no-any-return] raise AssertionError(object.__getattribute__(self, "msg")) @@ -3573,7 +3601,10 @@ def serialize(self, prefix: str, name: str) -> JsonDict: if prefix is not None: fullname = self.node.fullname if ( - fullname is not None + # See the comment above SymbolNode.fullname -- fullname can often be None, + # but for complex reasons it's annotated as being `Bogus[str]` instead of `str | None`, + # meaning mypy erroneously thinks the `fullname is not None` check here is redundant + fullname is not None # type: ignore[redundant-expr] and "." in fullname and fullname != prefix + "." + name and not (isinstance(self.node, Var) and self.node.from_module_getattr) @@ -3705,7 +3736,7 @@ def check_arg_kinds( if kind == ARG_POS: if is_var_arg or is_kw_arg or seen_named or seen_opt: fail( - "Required positional args may not appear " "after default, named or var args", + "Required positional args may not appear after default, named or var args", node, ) break diff --git a/mypy/options.py b/mypy/options.py index ac46b70f8ebe..3a08ff9455ee 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -3,15 +3,13 @@ import pprint import re import sys -from typing import TYPE_CHECKING, Any, Callable, Mapping, Pattern +from typing import Any, Callable, Dict, Mapping, Pattern from typing_extensions import Final from mypy import defaults +from mypy.errorcodes import ErrorCode, error_codes from mypy.util import get_class_descriptors, replace_object_state -if TYPE_CHECKING: - from mypy.errorcodes import ErrorCode - class BuildType: STANDARD: Final = 0 @@ -27,6 +25,8 @@ class BuildType: "always_true", "check_untyped_defs", "debug_cache", + "disable_error_code", + "disabled_error_codes", "disallow_any_decorated", "disallow_any_explicit", "disallow_any_expr", @@ -37,28 +37,40 @@ class BuildType: "disallow_untyped_calls", "disallow_untyped_decorators", "disallow_untyped_defs", - "follow_imports", + "enable_error_code", + "enabled_error_codes", "follow_imports_for_stubs", + "follow_imports", "ignore_errors", "ignore_missing_imports", + "implicit_optional", "implicit_reexport", "local_partial_types", "mypyc", - "no_implicit_optional", - "show_none_errors", "strict_concatenate", "strict_equality", "strict_optional", - "strict_optional_whitelist", "warn_no_return", "warn_return_any", "warn_unreachable", "warn_unused_ignores", } -OPTIONS_AFFECTING_CACHE: Final = (PER_MODULE_OPTIONS | {"platform", "bazel", "plugins"}) - { - "debug_cache" -} +OPTIONS_AFFECTING_CACHE: Final = ( + PER_MODULE_OPTIONS + | { + "platform", + "bazel", + "plugins", + "disable_bytearray_promotion", + "disable_memoryview_promotion", + } +) - {"debug_cache"} + +# Features that are currently incomplete/experimental +TYPE_VAR_TUPLE: Final = "TypeVarTuple" +UNPACK: Final = "Unpack" +INCOMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK)) class Options: @@ -77,6 +89,8 @@ def __init__(self) -> None: self.platform = sys.platform self.custom_typing_module: str | None = None self.custom_typeshed_dir: str | None = None + # The abspath() version of the above, we compute it once as an optimization. + self.abs_custom_typeshed_dir: str | None = None self.mypy_path: list[str] = [] self.report_dirs: dict[str, str] = {} # Show errors in PEP 561 packages/site-packages modules @@ -93,7 +107,7 @@ def __init__(self) -> None: # This allows definitions of packages without __init__.py and allows packages to span # multiple directories. This flag affects both import discovery and the association of # input files/modules/packages to the relevant file and fully qualified module name. - self.namespace_packages = False + self.namespace_packages = True # Use current directory and MYPYPATH to determine fully qualified module names of files # passed by automatically considering their subdirectories as packages. This is only # relevant if namespace packages are enabled, since otherwise examining __init__.py's is @@ -160,15 +174,8 @@ def __init__(self) -> None: self.color_output = True self.error_summary = True - # Files in which to allow strict-Optional related errors - # TODO: Kill this in favor of show_none_errors - self.strict_optional_whitelist: list[str] | None = None - - # Alternate way to show/hide strict-None-checking related errors - self.show_none_errors = True - - # Don't assume arguments with default values of None are Optional - self.no_implicit_optional = False + # Assume arguments with default values of None are Optional + self.implicit_optional = False # Don't re-export names unless they are imported with `from ... as ...` self.implicit_reexport = True @@ -220,6 +227,12 @@ def __init__(self) -> None: # supports globbing self.files: list[str] | None = None + # A list of packages for mypy to type check + self.packages: list[str] | None = None + + # A list of modules for mypy to type check + self.modules: list[str] | None = None + # Write junit.xml to given file self.junit_xml: str | None = None @@ -267,7 +280,8 @@ def __init__(self) -> None: self.dump_type_stats = False self.dump_inference_stats = False self.dump_build_stats = False - self.enable_incomplete_features = False + self.enable_incomplete_features = False # deprecated + self.enable_incomplete_feature: list[str] = [] self.timing_stats: str | None = None # -- test options -- @@ -281,7 +295,7 @@ def __init__(self) -> None: self.shadow_file: list[list[str]] | None = None self.show_column_numbers: bool = False self.show_error_end: bool = False - self.show_error_codes = False + self.hide_error_codes = False # Use soft word wrap and show trimmed source snippets with error location markers. self.pretty = False self.dump_graph = False @@ -301,6 +315,8 @@ def __init__(self) -> None: self.fast_exit = True # fast path for finding modules from source set self.fast_module_lookup = False + # Allow empty function bodies even if it is not safe, used for testing only. + self.allow_empty_bodies = False # Used to transform source code before parsing if not None # TODO: Make the type precise (AnyStr -> AnyStr) self.transform_source: Callable[[Any], Any] | None = None @@ -315,9 +331,14 @@ def __init__(self) -> None: # skip most errors after this many messages have been reported. # -1 means unlimited. self.many_errors_threshold = defaults.MANY_ERRORS_THRESHOLD - # Enable recursive type aliases (currently experimental) + # Disable recursive type aliases (currently experimental) + self.disable_recursive_aliases = False + # Deprecated reverse version of the above, do not use. self.enable_recursive_aliases = False + self.disable_bytearray_promotion = False + self.disable_memoryview_promotion = False + # To avoid breaking plugin compatibility, keep providing new_semantic_analyzer @property def new_semantic_analyzer(self) -> bool: @@ -347,6 +368,20 @@ def apply_changes(self, changes: dict[str, object]) -> Options: # This is the only option for which a per-module and a global # option sometimes beheave differently. new_options.ignore_missing_imports_per_module = True + + # These two act as overrides, so apply them when cloning. + # Similar to global codes enabling overrides disabling, so we start from latter. + new_options.disabled_error_codes = self.disabled_error_codes.copy() + new_options.enabled_error_codes = self.enabled_error_codes.copy() + for code_str in new_options.disable_error_code: + code = error_codes[code_str] + new_options.disabled_error_codes.add(code) + new_options.enabled_error_codes.discard(code) + for code_str in new_options.enable_error_code: + code = error_codes[code_str] + new_options.enabled_error_codes.add(code) + new_options.disabled_error_codes.discard(code) + return new_options def build_per_module_cache(self) -> None: @@ -446,4 +481,10 @@ def compile_glob(self, s: str) -> Pattern[str]: return re.compile(expr + "\\Z") def select_options_affecting_cache(self) -> Mapping[str, object]: - return {opt: getattr(self, opt) for opt in OPTIONS_AFFECTING_CACHE} + result: Dict[str, object] = {} + for opt in OPTIONS_AFFECTING_CACHE: + val = getattr(self, opt) + if opt in ("disabled_error_codes", "enabled_error_codes"): + val = sorted([code.code for code in val]) + result[opt] = val + return result diff --git a/mypy/partially_defined.py b/mypy/partially_defined.py new file mode 100644 index 000000000000..7d87315c23ad --- /dev/null +++ b/mypy/partially_defined.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +from mypy import checker +from mypy.messages import MessageBuilder +from mypy.nodes import ( + AssertStmt, + AssignmentExpr, + AssignmentStmt, + BreakStmt, + ContinueStmt, + DictionaryComprehension, + Expression, + ExpressionStmt, + ForStmt, + FuncDef, + FuncItem, + GeneratorExpr, + IfStmt, + ListExpr, + Lvalue, + MatchStmt, + NameExpr, + RaiseStmt, + ReturnStmt, + TupleExpr, + WhileStmt, + WithStmt, +) +from mypy.patterns import AsPattern, StarredPattern +from mypy.reachability import ALWAYS_TRUE, infer_pattern_value +from mypy.traverser import ExtendedTraverserVisitor +from mypy.types import Type, UninhabitedType + + +class BranchState: + """BranchState contains information about variable definition at the end of a branching statement. + `if` and `match` are examples of branching statements. + + `may_be_defined` contains variables that were defined in only some branches. + `must_be_defined` contains variables that were defined in all branches. + """ + + def __init__( + self, + must_be_defined: set[str] | None = None, + may_be_defined: set[str] | None = None, + skipped: bool = False, + ) -> None: + if may_be_defined is None: + may_be_defined = set() + if must_be_defined is None: + must_be_defined = set() + + self.may_be_defined = set(may_be_defined) + self.must_be_defined = set(must_be_defined) + self.skipped = skipped + + +class BranchStatement: + def __init__(self, initial_state: BranchState) -> None: + self.initial_state = initial_state + self.branches: list[BranchState] = [ + BranchState( + must_be_defined=self.initial_state.must_be_defined, + may_be_defined=self.initial_state.may_be_defined, + ) + ] + + def next_branch(self) -> None: + self.branches.append( + BranchState( + must_be_defined=self.initial_state.must_be_defined, + may_be_defined=self.initial_state.may_be_defined, + ) + ) + + def record_definition(self, name: str) -> None: + assert len(self.branches) > 0 + self.branches[-1].must_be_defined.add(name) + self.branches[-1].may_be_defined.discard(name) + + def record_nested_branch(self, state: BranchState) -> None: + assert len(self.branches) > 0 + current_branch = self.branches[-1] + if state.skipped: + current_branch.skipped = True + return + current_branch.must_be_defined.update(state.must_be_defined) + current_branch.may_be_defined.update(state.may_be_defined) + current_branch.may_be_defined.difference_update(current_branch.must_be_defined) + + def skip_branch(self) -> None: + assert len(self.branches) > 0 + self.branches[-1].skipped = True + + def is_possibly_undefined(self, name: str) -> bool: + assert len(self.branches) > 0 + return name in self.branches[-1].may_be_defined + + def done(self) -> BranchState: + branches = [b for b in self.branches if not b.skipped] + if len(branches) == 0: + return BranchState(skipped=True) + if len(branches) == 1: + return branches[0] + + # must_be_defined is a union of must_be_defined of all branches. + must_be_defined = set(branches[0].must_be_defined) + for b in branches[1:]: + must_be_defined.intersection_update(b.must_be_defined) + # may_be_defined are all variables that are not must be defined. + all_vars = set() + for b in branches: + all_vars.update(b.may_be_defined) + all_vars.update(b.must_be_defined) + may_be_defined = all_vars.difference(must_be_defined) + return BranchState(may_be_defined=may_be_defined, must_be_defined=must_be_defined) + + +class DefinedVariableTracker: + """DefinedVariableTracker manages the state and scope for the UndefinedVariablesVisitor.""" + + def __init__(self) -> None: + # There's always at least one scope. Within each scope, there's at least one "global" BranchingStatement. + self.scopes: list[list[BranchStatement]] = [[BranchStatement(BranchState())]] + + def _scope(self) -> list[BranchStatement]: + assert len(self.scopes) > 0 + return self.scopes[-1] + + def enter_scope(self) -> None: + assert len(self._scope()) > 0 + self.scopes.append([BranchStatement(self._scope()[-1].branches[-1])]) + + def exit_scope(self) -> None: + self.scopes.pop() + + def start_branch_statement(self) -> None: + assert len(self._scope()) > 0 + self._scope().append(BranchStatement(self._scope()[-1].branches[-1])) + + def next_branch(self) -> None: + assert len(self._scope()) > 1 + self._scope()[-1].next_branch() + + def end_branch_statement(self) -> None: + assert len(self._scope()) > 1 + result = self._scope().pop().done() + self._scope()[-1].record_nested_branch(result) + + def skip_branch(self) -> None: + # Only skip branch if we're outside of "root" branch statement. + if len(self._scope()) > 1: + self._scope()[-1].skip_branch() + + def record_declaration(self, name: str) -> None: + assert len(self.scopes) > 0 + assert len(self.scopes[-1]) > 0 + self._scope()[-1].record_definition(name) + + def is_possibly_undefined(self, name: str) -> bool: + assert len(self._scope()) > 0 + # A variable is undefined if it's in a set of `may_be_defined` but not in `must_be_defined`. + # Cases where a variable is not defined altogether are handled by semantic analyzer. + return self._scope()[-1].is_possibly_undefined(name) + + +class PartiallyDefinedVariableVisitor(ExtendedTraverserVisitor): + """Detect variables that are defined only part of the time. + + This visitor detects the following case: + if foo(): + x = 1 + print(x) # Error: "x" may be undefined. + + Note that this code does not detect variables not defined in any of the branches -- that is + handled by the semantic analyzer. + """ + + def __init__(self, msg: MessageBuilder, type_map: dict[Expression, Type]) -> None: + self.msg = msg + self.type_map = type_map + self.tracker = DefinedVariableTracker() + + def process_lvalue(self, lvalue: Lvalue | None) -> None: + if isinstance(lvalue, NameExpr): + self.tracker.record_declaration(lvalue.name) + elif isinstance(lvalue, (ListExpr, TupleExpr)): + for item in lvalue.items: + self.process_lvalue(item) + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + for lvalue in o.lvalues: + self.process_lvalue(lvalue) + super().visit_assignment_stmt(o) + + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + o.value.accept(self) + self.process_lvalue(o.target) + + def visit_if_stmt(self, o: IfStmt) -> None: + for e in o.expr: + e.accept(self) + self.tracker.start_branch_statement() + for b in o.body: + b.accept(self) + self.tracker.next_branch() + if o.else_body: + o.else_body.accept(self) + self.tracker.end_branch_statement() + + def visit_match_stmt(self, o: MatchStmt) -> None: + self.tracker.start_branch_statement() + o.subject.accept(self) + for i in range(len(o.patterns)): + pattern = o.patterns[i] + pattern.accept(self) + guard = o.guards[i] + if guard is not None: + guard.accept(self) + o.bodies[i].accept(self) + is_catchall = infer_pattern_value(pattern) == ALWAYS_TRUE + if not is_catchall: + self.tracker.next_branch() + self.tracker.end_branch_statement() + + def visit_func_def(self, o: FuncDef) -> None: + self.tracker.enter_scope() + super().visit_func_def(o) + self.tracker.exit_scope() + + def visit_func(self, o: FuncItem) -> None: + if o.arguments is not None: + for arg in o.arguments: + self.tracker.record_declaration(arg.variable.name) + super().visit_func(o) + + def visit_generator_expr(self, o: GeneratorExpr) -> None: + self.tracker.enter_scope() + for idx in o.indices: + self.process_lvalue(idx) + super().visit_generator_expr(o) + self.tracker.exit_scope() + + def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None: + self.tracker.enter_scope() + for idx in o.indices: + self.process_lvalue(idx) + super().visit_dictionary_comprehension(o) + self.tracker.exit_scope() + + def visit_for_stmt(self, o: ForStmt) -> None: + o.expr.accept(self) + self.process_lvalue(o.index) + o.index.accept(self) + self.tracker.start_branch_statement() + o.body.accept(self) + self.tracker.next_branch() + if o.else_body: + o.else_body.accept(self) + self.tracker.end_branch_statement() + + def visit_return_stmt(self, o: ReturnStmt) -> None: + super().visit_return_stmt(o) + self.tracker.skip_branch() + + def visit_assert_stmt(self, o: AssertStmt) -> None: + super().visit_assert_stmt(o) + if checker.is_false_literal(o.expr): + self.tracker.skip_branch() + + def visit_raise_stmt(self, o: RaiseStmt) -> None: + super().visit_raise_stmt(o) + self.tracker.skip_branch() + + def visit_continue_stmt(self, o: ContinueStmt) -> None: + super().visit_continue_stmt(o) + self.tracker.skip_branch() + + def visit_break_stmt(self, o: BreakStmt) -> None: + super().visit_break_stmt(o) + self.tracker.skip_branch() + + def visit_expression_stmt(self, o: ExpressionStmt) -> None: + if isinstance(self.type_map.get(o.expr, None), UninhabitedType): + self.tracker.skip_branch() + super().visit_expression_stmt(o) + + def visit_while_stmt(self, o: WhileStmt) -> None: + o.expr.accept(self) + self.tracker.start_branch_statement() + o.body.accept(self) + if not checker.is_true_literal(o.expr): + self.tracker.next_branch() + if o.else_body: + o.else_body.accept(self) + self.tracker.end_branch_statement() + + def visit_as_pattern(self, o: AsPattern) -> None: + if o.name is not None: + self.process_lvalue(o.name) + super().visit_as_pattern(o) + + def visit_starred_pattern(self, o: StarredPattern) -> None: + if o.capture is not None: + self.process_lvalue(o.capture) + super().visit_starred_pattern(o) + + def visit_name_expr(self, o: NameExpr) -> None: + if self.tracker.is_possibly_undefined(o.name): + self.msg.variable_may_be_undefined(o.name, o) + # We don't want to report the error on the same variable multiple times. + self.tracker.record_declaration(o.name) + super().visit_name_expr(o) + + def visit_with_stmt(self, o: WithStmt) -> None: + for expr, idx in zip(o.expr, o.target): + expr.accept(self) + self.process_lvalue(idx) + o.body.accept(self) diff --git a/mypy/plugin.py b/mypy/plugin.py index dc31130df991..00a2af82969f 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -407,6 +407,10 @@ def final_iteration(self) -> bool: def is_stub_file(self) -> bool: raise NotImplementedError + @abstractmethod + def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Type | None: + raise NotImplementedError + # A context for querying for configuration data about a module for # cache invalidation purposes. diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py index e180d435dc35..17f1794d8c75 100644 --- a/mypy/plugins/attrs.py +++ b/mypy/plugins/attrs.py @@ -75,7 +75,7 @@ SELF_TVAR_NAME: Final = "_AT" MAGIC_ATTR_NAME: Final = "__attrs_attrs__" -MAGIC_ATTR_CLS_NAME: Final = "_AttrsAttributes" # The namedtuple subclass name. +MAGIC_ATTR_CLS_NAME_TEMPLATE: Final = "__{}_AttrsAttributes__" # The tuple subclass pattern. class Converter: @@ -257,7 +257,7 @@ def attr_tag_callback(ctx: mypy.plugin.ClassDefContext) -> None: """Record that we have an attrs class in the main semantic analysis pass. The later pass implemented by attr_class_maker_callback will use this - to detect attrs lasses in base classes. + to detect attrs classes in base classes. """ # The value is ignored, only the existence matters. ctx.cls.info.metadata["attrs_tag"] = {} @@ -324,8 +324,8 @@ def attr_class_maker_callback( } adder = MethodAdder(ctx) - if init: - _add_init(ctx, attributes, adder) + # If __init__ is not being generated, attrs still generates it as __attrs_init__ instead. + _add_init(ctx, attributes, adder, "__init__" if init else "__attrs_init__") if order: _add_order(ctx, adder) if frozen: @@ -749,7 +749,10 @@ def _make_frozen(ctx: mypy.plugin.ClassDefContext, attributes: list[Attribute]) def _add_init( - ctx: mypy.plugin.ClassDefContext, attributes: list[Attribute], adder: MethodAdder + ctx: mypy.plugin.ClassDefContext, + attributes: list[Attribute], + adder: MethodAdder, + method_name: str, ) -> None: """Generate an __init__ method for the attributes and add it to the class.""" # Convert attributes to arguments with kw_only arguments at the end of @@ -777,7 +780,7 @@ def _add_init( for a in args: a.variable.type = AnyType(TypeOfAny.implementation_artifact) a.type_annotation = AnyType(TypeOfAny.implementation_artifact) - adder.add_method("__init__", args, NoneType()) + adder.add_method(method_name, args, NoneType()) def _add_attrs_magic_attribute( @@ -792,10 +795,11 @@ def _add_attrs_magic_attribute( "builtins.tuple", [ctx.api.named_type_or_none("attr.Attribute", [any_type]) or any_type] ) - ti = ctx.api.basic_new_typeinfo(MAGIC_ATTR_CLS_NAME, fallback_type, 0) - ti.is_named_tuple = True + attr_name = MAGIC_ATTR_CLS_NAME_TEMPLATE.format(ctx.cls.fullname.replace(".", "_")) + ti = ctx.api.basic_new_typeinfo(attr_name, fallback_type, 0) for (name, _), attr_type in zip(attrs, attributes_types): var = Var(name, attr_type) + var._fullname = name var.is_property = True proper_type = get_proper_type(attr_type) if isinstance(proper_type, Instance): @@ -803,14 +807,18 @@ def _add_attrs_magic_attribute( ti.names[name] = SymbolTableNode(MDEF, var, plugin_generated=True) attributes_type = Instance(ti, []) - # TODO: refactor using `add_attribute_to_class` - var = Var(name=MAGIC_ATTR_NAME, type=TupleType(attributes_types, fallback=attributes_type)) - var.info = ctx.cls.info - var.is_classvar = True - var._fullname = f"{ctx.cls.fullname}.{MAGIC_ATTR_CLS_NAME}" - var.allow_incompatible_override = True - ctx.cls.info.names[MAGIC_ATTR_NAME] = SymbolTableNode( - kind=MDEF, node=var, plugin_generated=True, no_serialize=True + # We need to stash the type of the magic attribute so it can be + # loaded on cached runs. + ctx.cls.info.names[attr_name] = SymbolTableNode(MDEF, ti, plugin_generated=True) + + add_attribute_to_class( + ctx.api, + ctx.cls, + MAGIC_ATTR_NAME, + TupleType(attributes_types, fallback=attributes_type), + fullname=f"{ctx.cls.fullname}.{attr_name}", + override_allow_incompatible=True, + is_classvar=True, ) diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index edcf8ea9a082..07cd5dc7de7f 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -9,6 +9,7 @@ Block, CallExpr, ClassDef, + Decorator, Expression, FuncDef, JsonDict, @@ -26,6 +27,7 @@ CallableType, Overloaded, Type, + TypeType, TypeVarType, deserialize_type, get_proper_type, @@ -102,6 +104,8 @@ def add_method( return_type: Type, self_type: Type | None = None, tvar_def: TypeVarType | None = None, + is_classmethod: bool = False, + is_staticmethod: bool = False, ) -> None: """ Adds a new method to a class. @@ -115,6 +119,8 @@ def add_method( return_type=return_type, self_type=self_type, tvar_def=tvar_def, + is_classmethod=is_classmethod, + is_staticmethod=is_staticmethod, ) @@ -126,8 +132,15 @@ def add_method_to_class( return_type: Type, self_type: Type | None = None, tvar_def: TypeVarType | None = None, + is_classmethod: bool = False, + is_staticmethod: bool = False, ) -> None: """Adds a new method to a class definition.""" + + assert not ( + is_classmethod is True and is_staticmethod is True + ), "Can't add a new method that's both staticmethod and classmethod." + info = cls.info # First remove any previously generated methods with the same name @@ -137,13 +150,21 @@ def add_method_to_class( if sym.plugin_generated and isinstance(sym.node, FuncDef): cls.defs.body.remove(sym.node) - self_type = self_type or fill_typevars(info) if isinstance(api, SemanticAnalyzerPluginInterface): function_type = api.named_type("builtins.function") else: function_type = api.named_generic_type("builtins.function", []) - args = [Argument(Var("self"), self_type, None, ARG_POS)] + args + if is_classmethod: + self_type = self_type or TypeType(fill_typevars(info)) + first = [Argument(Var("_cls"), self_type, None, ARG_POS, True)] + elif is_staticmethod: + first = [] + else: + self_type = self_type or fill_typevars(info) + first = [Argument(Var("self"), self_type, None, ARG_POS)] + args = first + args + arg_types, arg_names, arg_kinds = [], [], [] for arg in args: assert arg.type_annotation, "All arguments must be fully typed." @@ -158,6 +179,8 @@ def add_method_to_class( func = FuncDef(name, args, Block([PassStmt()])) func.info = info func.type = set_callable_name(signature, func) + func.is_class = is_classmethod + func.is_static = is_staticmethod func._fullname = info.fullname + "." + name func.line = info.line @@ -168,7 +191,21 @@ def add_method_to_class( r_name = get_unique_redefinition_name(name, info.names) info.names[r_name] = info.names[name] - info.names[name] = SymbolTableNode(MDEF, func, plugin_generated=True) + # Add decorator for is_staticmethod. It's unnecessary for is_classmethod. + if is_staticmethod: + func.is_decorated = True + v = Var(name, func.type) + v.info = info + v._fullname = func._fullname + v.is_staticmethod = True + dec = Decorator(func, [], v) + dec.line = info.line + sym = SymbolTableNode(MDEF, dec) + else: + sym = SymbolTableNode(MDEF, func) + sym.plugin_generated = True + info.names[name] = sym + info.defn.defs.body.append(func) @@ -180,6 +217,8 @@ def add_attribute_to_class( final: bool = False, no_serialize: bool = False, override_allow_incompatible: bool = False, + fullname: str | None = None, + is_classvar: bool = False, ) -> None: """ Adds a new attribute to a class definition. @@ -197,11 +236,17 @@ def add_attribute_to_class( node = Var(name, typ) node.info = info node.is_final = final + node.is_classvar = is_classvar if name in ALLOW_INCOMPATIBLE_OVERRIDE: node.allow_incompatible_override = True else: node.allow_incompatible_override = override_allow_incompatible - node._fullname = info.fullname + "." + name + + if fullname: + node._fullname = fullname + else: + node._fullname = info.fullname + "." + name + info.names[name] = SymbolTableNode( MDEF, node, plugin_generated=True, no_serialize=no_serialize ) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 095967dc3fa1..26bc8ae80fdb 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -244,10 +244,20 @@ def transform(self) -> bool: tvar_def=order_tvar_def, ) + parent_decorator_arguments = [] + for parent in info.mro[1:-1]: + parent_args = parent.metadata.get("dataclass") + if parent_args: + parent_decorator_arguments.append(parent_args) + if decorator_arguments["frozen"]: + if any(not parent["frozen"] for parent in parent_decorator_arguments): + ctx.api.fail("Cannot inherit frozen dataclass from a non-frozen one", info) self._propertize_callables(attributes, settable=False) self._freeze(attributes) else: + if any(parent["frozen"] for parent in parent_decorator_arguments): + ctx.api.fail("Cannot inherit non-frozen dataclass from a frozen one", info) self._propertize_callables(attributes) if decorator_arguments["slots"]: @@ -340,11 +350,51 @@ def collect_attributes(self) -> list[DataclassAttribute] | None: Return None if some dataclass base class hasn't been processed yet and thus we'll need to ask for another pass. """ - # First, collect attributes belonging to the current class. ctx = self._ctx cls = self._ctx.cls - attrs: list[DataclassAttribute] = [] - known_attrs: set[str] = set() + + # First, collect attributes belonging to any class in the MRO, ignoring duplicates. + # + # We iterate through the MRO in reverse because attrs defined in the parent must appear + # earlier in the attributes list than attrs defined in the child. See: + # https://docs.python.org/3/library/dataclasses.html#inheritance + # + # However, we also want attributes defined in the subtype to override ones defined + # in the parent. We can implement this via a dict without disrupting the attr order + # because dicts preserve insertion order in Python 3.7+. + found_attrs: dict[str, DataclassAttribute] = {} + found_dataclass_supertype = False + for info in reversed(cls.info.mro[1:-1]): + if "dataclass_tag" in info.metadata and "dataclass" not in info.metadata: + # We haven't processed the base class yet. Need another pass. + return None + if "dataclass" not in info.metadata: + continue + + # Each class depends on the set of attributes in its dataclass ancestors. + ctx.api.add_plugin_dependency(make_wildcard_trigger(info.fullname)) + found_dataclass_supertype = True + + for data in info.metadata["dataclass"]["attributes"]: + name: str = data["name"] + + attr = DataclassAttribute.deserialize(info, data, ctx.api) + # TODO: We shouldn't be performing type operations during the main + # semantic analysis pass, since some TypeInfo attributes might + # still be in flux. This should be performed in a later phase. + with state.strict_optional_set(ctx.api.options.strict_optional): + attr.expand_typevar_from_subtype(ctx.cls.info) + found_attrs[name] = attr + + sym_node = cls.info.names.get(name) + if sym_node and sym_node.node and not isinstance(sym_node.node, Var): + ctx.api.fail( + "Dataclass attribute may only be overridden by another attribute", + sym_node.node, + ) + + # Second, collect attributes belonging to the current class. + current_attr_names: set[str] = set() kw_only = _get_decorator_bool_argument(ctx, "kw_only", False) for stmt in cls.defs.body: # Any assignment that doesn't use the new type declaration @@ -368,7 +418,7 @@ def collect_attributes(self) -> list[DataclassAttribute] | None: if isinstance(node, TypeAlias): ctx.api.fail( - ("Type aliases inside dataclass definitions " "are not supported at runtime"), + ("Type aliases inside dataclass definitions are not supported at runtime"), node, ) # Skip processing this node. This doesn't match the runtime behaviour, @@ -425,66 +475,43 @@ def collect_attributes(self) -> list[DataclassAttribute] | None: if field_kw_only_param is not None: is_kw_only = bool(ctx.api.parse_bool(field_kw_only_param)) - known_attrs.add(lhs.name) - attrs.append( - DataclassAttribute( - name=lhs.name, - is_in_init=is_in_init, - is_init_var=is_init_var, - has_default=has_default, - line=stmt.line, - column=stmt.column, - type=sym.type, - info=cls.info, - kw_only=is_kw_only, - ) + if sym.type is None and node.is_final and node.is_inferred: + # This is a special case, assignment like x: Final = 42 is classified + # annotated above, but mypy strips the `Final` turning it into x = 42. + # We do not support inferred types in dataclasses, so we can try inferring + # type for simple literals, and otherwise require an explicit type + # argument for Final[...]. + typ = ctx.api.analyze_simple_literal_type(stmt.rvalue, is_final=True) + if typ: + node.type = typ + else: + ctx.api.fail( + "Need type argument for Final[...] with non-literal default in dataclass", + stmt, + ) + node.type = AnyType(TypeOfAny.from_error) + + current_attr_names.add(lhs.name) + found_attrs[lhs.name] = DataclassAttribute( + name=lhs.name, + is_in_init=is_in_init, + is_init_var=is_init_var, + has_default=has_default, + line=stmt.line, + column=stmt.column, + type=sym.type, + info=cls.info, + kw_only=is_kw_only, ) - # Next, collect attributes belonging to any class in the MRO - # as long as those attributes weren't already collected. This - # makes it possible to overwrite attributes in subclasses. - # copy() because we potentially modify all_attrs below and if this code requires debugging - # we'll have unmodified attrs laying around. - all_attrs = attrs.copy() - for info in cls.info.mro[1:-1]: - if "dataclass_tag" in info.metadata and "dataclass" not in info.metadata: - # We haven't processed the base class yet. Need another pass. - return None - if "dataclass" not in info.metadata: - continue - - super_attrs = [] - # Each class depends on the set of attributes in its dataclass ancestors. - ctx.api.add_plugin_dependency(make_wildcard_trigger(info.fullname)) - - for data in info.metadata["dataclass"]["attributes"]: - name: str = data["name"] - if name not in known_attrs: - attr = DataclassAttribute.deserialize(info, data, ctx.api) - # TODO: We shouldn't be performing type operations during the main - # semantic analysis pass, since some TypeInfo attributes might - # still be in flux. This should be performed in a later phase. - with state.strict_optional_set(ctx.api.options.strict_optional): - attr.expand_typevar_from_subtype(ctx.cls.info) - known_attrs.add(name) - super_attrs.append(attr) - elif all_attrs: - # How early in the attribute list an attribute appears is determined by the - # reverse MRO, not simply MRO. - # See https://docs.python.org/3/library/dataclasses.html#inheritance for - # details. - for attr in all_attrs: - if attr.name == name: - all_attrs.remove(attr) - super_attrs.append(attr) - break - all_attrs = super_attrs + all_attrs + all_attrs = list(found_attrs.values()) + if found_dataclass_supertype: all_attrs.sort(key=lambda a: a.kw_only) - # Ensure that arguments without a default don't follow - # arguments that have a default. + # Third, ensure that arguments without a default don't follow + # arguments that have a default and that the KW_ONLY sentinel + # is only provided once. found_default = False - # Ensure that the KW_ONLY sentinel is only provided once found_kw_sentinel = False for attr in all_attrs: # If we find any attribute that is_in_init, not kw_only, and that @@ -493,17 +520,20 @@ def collect_attributes(self) -> list[DataclassAttribute] | None: if found_default and attr.is_in_init and not attr.has_default and not attr.kw_only: # If the issue comes from merging different classes, report it # at the class definition point. - context = Context(line=attr.line, column=attr.column) if attr in attrs else ctx.cls + context: Context = ctx.cls + if attr.name in current_attr_names: + context = Context(line=attr.line, column=attr.column) ctx.api.fail( "Attributes without a default cannot follow attributes with one", context ) found_default = found_default or (attr.has_default and attr.is_in_init) if found_kw_sentinel and self._is_kw_only_type(attr.type): - context = Context(line=attr.line, column=attr.column) if attr in attrs else ctx.cls + context = ctx.cls + if attr.name in current_attr_names: + context = Context(line=attr.line, column=attr.column) ctx.api.fail("There may not be more than one field with the KW_ONLY type", context) found_kw_sentinel = found_kw_sentinel or self._is_kw_only_type(attr.type) - return all_attrs def _freeze(self, attributes: list[DataclassAttribute]) -> None: @@ -515,8 +545,8 @@ def _freeze(self, attributes: list[DataclassAttribute]) -> None: sym_node = info.names.get(attr.name) if sym_node is not None: var = sym_node.node - assert isinstance(var, Var) - var.is_property = True + if isinstance(var, Var): + var.is_property = True else: var = attr.to_var() var.info = info @@ -563,6 +593,7 @@ def _add_dataclass_fields_magic_attribute(self) -> None: var = Var(name=attr_name, type=attr_type) var.info = self._ctx.cls.info var._fullname = self._ctx.cls.info.fullname + "." + attr_name + var.is_classvar = True self._ctx.cls.info.names[attr_name] = SymbolTableNode( kind=MDEF, node=var, plugin_generated=True ) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 5ec37230b5ed..04971868e8f4 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -39,9 +39,7 @@ class DefaultPlugin(Plugin): def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: from mypy.plugins import ctypes, singledispatch - if fullname in ("contextlib.contextmanager", "contextlib.asynccontextmanager"): - return contextmanager_callback - elif fullname == "ctypes.Array": + if fullname == "ctypes.Array": return ctypes.array_constructor_callback elif fullname == "functools.singledispatch": return singledispatch.create_singledispatch_function_callback @@ -148,25 +146,6 @@ def get_class_decorator_hook_2( return None -def contextmanager_callback(ctx: FunctionContext) -> Type: - """Infer a better return type for 'contextlib.contextmanager'.""" - # Be defensive, just in case. - if ctx.arg_types and len(ctx.arg_types[0]) == 1: - arg_type = get_proper_type(ctx.arg_types[0][0]) - default_return = get_proper_type(ctx.default_return_type) - if isinstance(arg_type, CallableType) and isinstance(default_return, CallableType): - # The stub signature doesn't preserve information about arguments so - # add them back here. - return default_return.copy_modified( - arg_types=arg_type.arg_types, - arg_kinds=arg_type.arg_kinds, - arg_names=arg_type.arg_names, - variables=arg_type.variables, - is_ellipsis_args=arg_type.is_ellipsis_args, - ) - return ctx.default_return_type - - def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType: """Try to infer a better signature type for TypedDict.get. diff --git a/mypy/plugins/singledispatch.py b/mypy/plugins/singledispatch.py index e6009e64f789..cd6a3a9fa1cc 100644 --- a/mypy/plugins/singledispatch.py +++ b/mypy/plugins/singledispatch.py @@ -40,7 +40,7 @@ class RegisterCallableInfo(NamedTuple): def get_singledispatch_info(typ: Instance) -> SingledispatchTypeVars | None: if len(typ.args) == 2: - return SingledispatchTypeVars(*typ.args) # type: ignore + return SingledispatchTypeVars(*typ.args) # type: ignore[arg-type] return None @@ -200,7 +200,7 @@ def call_singledispatch_function_after_register_argument(ctx: MethodContext) -> """Called on the function after passing a type to register""" register_callable = ctx.type if isinstance(register_callable, Instance): - type_args = RegisterCallableInfo(*register_callable.args) # type: ignore + type_args = RegisterCallableInfo(*register_callable.args) # type: ignore[arg-type] func = get_first_arg(ctx.arg_types) if func is not None: register_function(ctx, type_args.singledispatch_obj, func, type_args.register_type) diff --git a/mypy/reachability.py b/mypy/reachability.py index c4611a13d1af..8602fc645e2b 100644 --- a/mypy/reachability.py +++ b/mypy/reachability.py @@ -13,6 +13,7 @@ CallExpr, ComparisonExpr, Expression, + FuncDef, IfStmt, Import, ImportAll, @@ -274,7 +275,7 @@ def fixed_comparison(left: Targ, op: str, right: Targ) -> int: return TRUTH_VALUE_UNKNOWN -def contains_int_or_tuple_of_ints(expr: Expression) -> None | int | tuple[int] | tuple[int, ...]: +def contains_int_or_tuple_of_ints(expr: Expression) -> None | int | tuple[int, ...]: if isinstance(expr, IntExpr): return expr.value if isinstance(expr, TupleExpr): @@ -357,3 +358,6 @@ def visit_import_from(self, node: ImportFrom) -> None: def visit_import_all(self, node: ImportAll) -> None: node.is_mypy_only = True + + def visit_func_def(self, node: FuncDef) -> None: + node.is_mypy_only = True diff --git a/mypy/report.py b/mypy/report.py index 183d0390e2c9..37b7497f1371 100644 --- a/mypy/report.py +++ b/mypy/report.py @@ -10,7 +10,6 @@ import sys import time import tokenize -import typing from abc import ABCMeta, abstractmethod from operator import attrgetter from typing import Any, Callable, Dict, Iterator, Tuple, cast @@ -26,7 +25,7 @@ from mypy.version import __version__ try: - from lxml import etree # type: ignore + from lxml import etree # type: ignore[import] LXML_INSTALLED = True except ImportError: @@ -141,8 +140,12 @@ def should_skip_path(path: str) -> bool: def iterate_python_lines(path: str) -> Iterator[tuple[int, str]]: """Return an iterator over (line number, line text) from a Python file.""" - with tokenize.open(path) as input_file: - yield from enumerate(input_file, 1) + try: + with tokenize.open(path) as input_file: + yield from enumerate(input_file, 1) + except IsADirectoryError: + # can happen with namespace packages + pass class FuncCounterVisitor(TraverserVisitor): @@ -211,7 +214,7 @@ class AnyExpressionsReporter(AbstractReporter): def __init__(self, reports: Reports, output_dir: str) -> None: super().__init__(reports, output_dir) self.counts: dict[str, tuple[int, int]] = {} - self.any_types_counter: dict[str, typing.Counter[int]] = {} + self.any_types_counter: dict[str, collections.Counter[int]] = {} def on_file( self, @@ -286,7 +289,7 @@ def _report_any_exprs(self) -> None: self._write_out_report("any-exprs.txt", column_names, rows, total_row) def _report_types_of_anys(self) -> None: - total_counter: typing.Counter[int] = collections.Counter() + total_counter: collections.Counter[int] = collections.Counter() for counter in self.any_types_counter.values(): for any_type, value in counter.items(): total_counter[any_type] += value @@ -373,7 +376,7 @@ def visit_func_def(self, defn: FuncDef) -> None: if cur_indent is None: # Consume the line, but don't mark it as belonging to the function yet. cur_line += 1 - elif start_indent is not None and cur_indent > start_indent: + elif cur_indent > start_indent: # A non-blank line that belongs to the function. cur_line += 1 end_line = cur_line @@ -528,7 +531,7 @@ def on_file( def _get_any_info_for_line(visitor: stats.StatisticsVisitor, lineno: int) -> str: if lineno in visitor.any_line_map: result = "Any Types on this line: " - counter: typing.Counter[int] = collections.Counter() + counter: collections.Counter[int] = collections.Counter() for typ in visitor.any_line_map[lineno]: counter[typ.type_of_any] += 1 for any_type, occurrences in counter.items(): diff --git a/mypy/semanal.py b/mypy/semanal.py index 08456c9ad845..b8f708b22a92 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -51,7 +51,7 @@ from __future__ import annotations from contextlib import contextmanager -from typing import Any, Callable, Iterable, Iterator, List, Optional, Set, TypeVar, cast +from typing import Any, Callable, Iterable, Iterator, List, TypeVar, cast from typing_extensions import Final, TypeAlias as _TypeAlias from mypy import errorcodes as codes, message_registry @@ -69,6 +69,8 @@ from mypy.nodes import ( ARG_NAMED, ARG_POS, + ARG_STAR, + ARG_STAR2, CONTRAVARIANT, COVARIANT, GDEF, @@ -77,6 +79,7 @@ IS_ABSTRACT, LDEF, MDEF, + NOT_ABSTRACT, REVEAL_LOCALS, REVEAL_TYPE, RUNTIME_PROTOCOL_DECOS, @@ -177,7 +180,7 @@ type_aliases_source_versions, typing_extensions_aliases, ) -from mypy.options import Options +from mypy.options import TYPE_VAR_TUPLE, Options from mypy.patterns import ( AsPattern, ClassPattern, @@ -263,6 +266,7 @@ TypeVarLikeType, TypeVarType, UnboundType, + UnpackType, get_proper_type, get_proper_types, invalid_recursive_alias, @@ -605,14 +609,18 @@ def add_implicit_module_attrs(self, file_node: MypyFile) -> None: if not sym: continue node = sym.node - assert isinstance(node, TypeInfo) + if not isinstance(node, TypeInfo): + self.defer(node) + return typ = Instance(node, [self.str_type()]) elif name == "__annotations__": sym = self.lookup_qualified("__builtins__.dict", Context(), suppress_errors=True) if not sym: continue node = sym.node - assert isinstance(node, TypeInfo) + if not isinstance(node, TypeInfo): + self.defer(node) + return typ = Instance(node, [self.str_type(), AnyType(TypeOfAny.special_form)]) else: assert t is not None, f"type should be specified for {name}" @@ -731,12 +739,14 @@ def file_context( """ scope = self.scope self.options = options - self.errors.set_file(file_node.path, file_node.fullname, scope=scope) + self.errors.set_file(file_node.path, file_node.fullname, scope=scope, options=options) self.cur_mod_node = file_node self.cur_mod_id = file_node.fullname with scope.module_scope(self.cur_mod_id): self._is_stub_file = file_node.path.lower().endswith(".pyi") - self._is_typeshed_stub_file = is_typeshed_file(file_node.path) + self._is_typeshed_stub_file = is_typeshed_file( + options.abs_custom_typeshed_dir, file_node.path + ) self.globals = file_node.names self.tvar_scope = TypeVarLikeScope() @@ -830,9 +840,12 @@ def analyze_func_def(self, defn: FuncDef) -> None: self.defer(defn) return assert isinstance(result, ProperType) + if isinstance(result, CallableType): + result = self.remove_unpack_kwargs(defn, result) defn.type = result self.add_type_alias_deps(analyzer.aliases_used) self.check_function_signature(defn) + self.check_paramspec_definition(defn) if isinstance(defn, FuncDef): assert isinstance(defn.type, CallableType) defn.type = set_callable_name(defn.type, defn) @@ -852,6 +865,12 @@ def analyze_func_def(self, defn: FuncDef) -> None: and is_trivial_body(defn.body) ): defn.abstract_status = IMPLICITLY_ABSTRACT + if ( + is_trivial_body(defn.body) + and not self.is_stub_file + and defn.abstract_status != NOT_ABSTRACT + ): + defn.is_trivial_body = True if ( defn.is_coroutine @@ -872,6 +891,29 @@ def analyze_func_def(self, defn: FuncDef) -> None: defn.type = defn.type.copy_modified(ret_type=ret_type) self.wrapped_coro_return_types[defn] = defn.type + def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType: + if not typ.arg_kinds or typ.arg_kinds[-1] is not ArgKind.ARG_STAR2: + return typ + last_type = get_proper_type(typ.arg_types[-1]) + if not isinstance(last_type, UnpackType): + return typ + last_type = get_proper_type(last_type.type) + if not isinstance(last_type, TypedDictType): + self.fail("Unpack item in ** argument must be a TypedDict", defn) + new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] + return typ.copy_modified(arg_types=new_arg_types) + overlap = set(typ.arg_names) & set(last_type.items) + # It is OK for TypedDict to have a key named 'kwargs'. + overlap.discard(typ.arg_names[-1]) + if overlap: + overlapped = ", ".join([f'"{name}"' for name in overlap]) + self.fail(f"Overlap between argument names and ** TypedDict items: {overlapped}", defn) + new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)] + return typ.copy_modified(arg_types=new_arg_types) + # OK, everything looks right now, mark the callable type as using unpack. + new_arg_types = typ.arg_types[:-1] + [last_type] + return typ.copy_modified(arg_types=new_arg_types, unpack_kwargs=True) + def prepare_method_signature(self, func: FuncDef, info: TypeInfo) -> None: """Check basic signature validity and tweak annotation of self/cls argument.""" # Only non-static methods are special. @@ -1006,6 +1048,8 @@ def process_overload_impl(self, defn: OverloadedFuncDef) -> None: assert self.type is not None if self.type.is_protocol: impl.abstract_status = IMPLICITLY_ABSTRACT + if impl.abstract_status != NOT_ABSTRACT: + impl.is_trivial_body = True def analyze_overload_sigs_and_impl( self, defn: OverloadedFuncDef @@ -1069,7 +1113,7 @@ def handle_missing_overload_decorators( ) else: self.fail( - "The implementation for an overloaded function " "must come last", + "The implementation for an overloaded function must come last", defn.items[idx], ) else: @@ -1093,6 +1137,7 @@ def handle_missing_overload_implementation(self, defn: OverloadedFuncDef) -> Non else: item.abstract_status = IS_ABSTRACT else: + # TODO: also allow omitting an implementation for abstract methods in ABCs? self.fail( "An overloaded function outside a stub file must have an implementation", defn, @@ -1215,8 +1260,9 @@ def analyze_function_body(self, defn: FuncItem) -> None: self.function_stack.pop() def check_classvar_in_signature(self, typ: ProperType) -> None: + t: ProperType if isinstance(typ, Overloaded): - for t in typ.items: # type: ProperType + for t in typ.items: self.check_classvar_in_signature(t) return if not isinstance(typ, CallableType): @@ -1239,6 +1285,64 @@ def check_function_signature(self, fdef: FuncItem) -> None: elif len(sig.arg_types) > len(fdef.arguments): self.fail("Type signature has too many arguments", fdef, blocker=True) + def check_paramspec_definition(self, defn: FuncDef) -> None: + func = defn.type + assert isinstance(func, CallableType) + + if not any(isinstance(var, ParamSpecType) for var in func.variables): + return # Function does not have param spec variables + + args = func.var_arg() + kwargs = func.kw_arg() + if args is None and kwargs is None: + return # Looks like this function does not have starred args + + args_defn_type = None + kwargs_defn_type = None + for arg_def, arg_kind in zip(defn.arguments, defn.arg_kinds): + if arg_kind == ARG_STAR: + args_defn_type = arg_def.type_annotation + elif arg_kind == ARG_STAR2: + kwargs_defn_type = arg_def.type_annotation + + # This may happen on invalid `ParamSpec` args / kwargs definition, + # type analyzer sets types of arguments to `Any`, but keeps + # definition types as `UnboundType` for now. + if not ( + (isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args")) + or ( + isinstance(kwargs_defn_type, UnboundType) + and kwargs_defn_type.name.endswith(".kwargs") + ) + ): + # Looks like both `*args` and `**kwargs` are not `ParamSpec` + # It might be something else, skipping. + return + + args_type = args.typ if args is not None else None + kwargs_type = kwargs.typ if kwargs is not None else None + + if ( + not isinstance(args_type, ParamSpecType) + or not isinstance(kwargs_type, ParamSpecType) + or args_type.name != kwargs_type.name + ): + if isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args"): + param_name = args_defn_type.name.split(".")[0] + elif isinstance(kwargs_defn_type, UnboundType) and kwargs_defn_type.name.endswith( + ".kwargs" + ): + param_name = kwargs_defn_type.name.split(".")[0] + else: + # Fallback for cases that probably should not ever happen: + param_name = "P" + + self.fail( + f'ParamSpec must have "*args" typed as "{param_name}.args" and "**kwargs" typed as "{param_name}.kwargs"', + func, + code=codes.VALID_TYPE, + ) + def visit_decorator(self, dec: Decorator) -> None: self.statement = dec # TODO: better don't modify them at all. @@ -1347,7 +1451,7 @@ def analyze_class(self, defn: ClassDef) -> None: defn.base_type_exprs.extend(defn.removed_base_type_exprs) defn.removed_base_type_exprs.clear() - self.update_metaclass(defn) + self.infer_metaclass_and_bases_from_compat_helpers(defn) bases = defn.base_type_exprs bases, tvar_defs, is_protocol = self.clean_up_bases_and_infer_type_variables( @@ -1363,20 +1467,25 @@ def analyze_class(self, defn: ClassDef) -> None: self.defer() self.analyze_class_keywords(defn) - result = self.analyze_base_classes(bases) - - if result is None or self.found_incomplete_ref(tag): + bases_result = self.analyze_base_classes(bases) + if bases_result is None or self.found_incomplete_ref(tag): # Something was incomplete. Defer current target. self.mark_incomplete(defn.name, defn) return - base_types, base_error = result + base_types, base_error = bases_result if any(isinstance(base, PlaceholderType) for base, _ in base_types): # We need to know the TypeInfo of each base to construct the MRO. Placeholder types # are okay in nested positions, since they can't affect the MRO. self.mark_incomplete(defn.name, defn) return + declared_metaclass, should_defer = self.get_declared_metaclass(defn.name, defn.metaclass) + if should_defer or self.found_incomplete_ref(tag): + # Metaclass was not ready. Defer current target. + self.mark_incomplete(defn.name, defn) + return + if self.analyze_typeddict_classdef(defn): if defn.info: self.setup_type_vars(defn, tvar_defs) @@ -1395,7 +1504,7 @@ def analyze_class(self, defn: ClassDef) -> None: with self.scope.class_scope(defn.info): self.configure_base_classes(defn, base_types) defn.info.is_protocol = is_protocol - self.analyze_metaclass(defn) + self.recalculate_metaclass(defn, declared_metaclass) defn.info.runtime_protocol = False for decorator in defn.decorators: self.analyze_class_decorator(defn, decorator) @@ -1442,8 +1551,8 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: for decorator in defn.decorators: decorator.accept(self) if isinstance(decorator, RefExpr): - if decorator.fullname in FINAL_DECORATOR_NAMES: - self.fail("@final cannot be used with TypedDict", decorator) + if decorator.fullname in FINAL_DECORATOR_NAMES and info is not None: + info.is_final = True if info is None: self.mark_incomplete(defn.name, defn) else: @@ -1463,7 +1572,8 @@ def analyze_namedtuple_classdef( ): # Don't reprocess everything. We just need to process methods defined # in the named tuple class body. - is_named_tuple, info = True, defn.info # type: bool, Optional[TypeInfo] + is_named_tuple = True + info: TypeInfo | None = defn.info else: is_named_tuple, info = self.named_tuple_analyzer.analyze_namedtuple_classdef( defn, self.is_stub_file, self.is_func_scope() @@ -1645,10 +1755,16 @@ def analyze_class_typevar_declaration(self, base: Type) -> tuple[TypeVarLikeList ): is_proto = sym.node.fullname != "typing.Generic" tvars: TypeVarLikeList = [] + have_type_var_tuple = False for arg in unbound.args: tag = self.track_incomplete_refs() tvar = self.analyze_unbound_tvar(arg) if tvar: + if isinstance(tvar[1], TypeVarTupleExpr): + if have_type_var_tuple: + self.fail("Can only use one type var tuple in a class def", base) + continue + have_type_var_tuple = True tvars.append(tvar) elif not self.found_incomplete_ref(tag): self.fail("Free type variable expected in %s[...]" % sym.node.name, base) @@ -1667,11 +1783,19 @@ def analyze_unbound_tvar(self, t: Type) -> tuple[str, TypeVarLikeExpr] | None: # It's bound by our type variable scope return None return unbound.name, sym.node - if sym and isinstance(sym.node, TypeVarTupleExpr): - if sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): - # It's bound by our type variable scope + if sym and sym.fullname == "typing_extensions.Unpack": + inner_t = unbound.args[0] + if not isinstance(inner_t, UnboundType): return None - return unbound.name, sym.node + inner_unbound = inner_t + inner_sym = self.lookup_qualified(inner_unbound.name, inner_unbound) + if inner_sym and isinstance(inner_sym.node, PlaceholderNode): + self.record_incomplete_ref() + if inner_sym and isinstance(inner_sym.node, TypeVarTupleExpr): + if inner_sym.fullname and not self.tvar_scope.allow_binding(inner_sym.fullname): + # It's bound by our type variable scope + return None + return inner_unbound.name, inner_sym.node if sym is None or not isinstance(sym.node, TypeVarExpr): return None elif sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): @@ -1940,7 +2064,7 @@ def calculate_class_mro( if hook: hook(ClassDefContext(defn, FakeExpression(), self)) - def update_metaclass(self, defn: ClassDef) -> None: + def infer_metaclass_and_bases_from_compat_helpers(self, defn: ClassDef) -> None: """Lookup for special metaclass declarations, and update defn fields accordingly. * six.with_metaclass(M, B1, B2, ...) @@ -2018,30 +2142,38 @@ def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: visited.add(base.type) return False - def analyze_metaclass(self, defn: ClassDef) -> None: - if defn.metaclass: + def get_declared_metaclass( + self, name: str, metaclass_expr: Expression | None + ) -> tuple[Instance | None, bool]: + """Returns either metaclass instance or boolean whether we should defer.""" + declared_metaclass = None + if metaclass_expr: metaclass_name = None - if isinstance(defn.metaclass, NameExpr): - metaclass_name = defn.metaclass.name - elif isinstance(defn.metaclass, MemberExpr): - metaclass_name = get_member_expr_fullname(defn.metaclass) + if isinstance(metaclass_expr, NameExpr): + metaclass_name = metaclass_expr.name + elif isinstance(metaclass_expr, MemberExpr): + metaclass_name = get_member_expr_fullname(metaclass_expr) if metaclass_name is None: - self.fail(f'Dynamic metaclass not supported for "{defn.name}"', defn.metaclass) - return - sym = self.lookup_qualified(metaclass_name, defn.metaclass) + self.fail(f'Dynamic metaclass not supported for "{name}"', metaclass_expr) + return None, False + sym = self.lookup_qualified(metaclass_name, metaclass_expr) if sym is None: # Probably a name error - it is already handled elsewhere - return + return None, False if isinstance(sym.node, Var) and isinstance(get_proper_type(sym.node.type), AnyType): - # 'Any' metaclass -- just ignore it. - # - # TODO: A better approach would be to record this information - # and assume that the type object supports arbitrary - # attributes, similar to an 'Any' base class. - return + # Create a fake TypeInfo that fallbacks to `Any`, basically allowing + # all the attributes. Same thing as we do for `Any` base class. + any_info = self.make_empty_type_info(ClassDef(sym.node.name, Block([]))) + any_info.fallback_to_any = True + any_info._fullname = sym.node.fullname + if self.options.disallow_subclassing_any: + self.fail( + f'Class cannot use "{any_info.fullname}" as a metaclass (has type "Any")', + metaclass_expr, + ) + return Instance(any_info, []), False if isinstance(sym.node, PlaceholderNode): - self.defer(defn) - return + return None, True # defer later in the caller # Support type aliases, like `_Meta: TypeAlias = type` if ( @@ -2055,16 +2187,20 @@ def analyze_metaclass(self, defn: ClassDef) -> None: metaclass_info = sym.node if not isinstance(metaclass_info, TypeInfo) or metaclass_info.tuple_type is not None: - self.fail(f'Invalid metaclass "{metaclass_name}"', defn.metaclass) - return + self.fail(f'Invalid metaclass "{metaclass_name}"', metaclass_expr) + return None, False if not metaclass_info.is_metaclass(): self.fail( - 'Metaclasses not inheriting from "type" are not supported', defn.metaclass + 'Metaclasses not inheriting from "type" are not supported', metaclass_expr ) - return + return None, False inst = fill_typevars(metaclass_info) assert isinstance(inst, Instance) - defn.info.declared_metaclass = inst + declared_metaclass = inst + return declared_metaclass, False + + def recalculate_metaclass(self, defn: ClassDef, declared_metaclass: Instance | None) -> None: + defn.info.declared_metaclass = declared_metaclass defn.info.metaclass_type = defn.info.calculate_metaclass_type() if any(info.is_protocol for info in defn.info.mro): if ( @@ -2076,16 +2212,10 @@ def analyze_metaclass(self, defn: ClassDef) -> None: abc_meta = self.named_type_or_none("abc.ABCMeta", []) if abc_meta is not None: # May be None in tests with incomplete lib-stub. defn.info.metaclass_type = abc_meta - if defn.info.metaclass_type is None: - # Inconsistency may happen due to multiple baseclasses even in classes that - # do not declare explicit metaclass, but it's harder to catch at this stage - if defn.metaclass is not None: - self.fail(f'Inconsistent metaclass structure for "{defn.name}"', defn) - else: - if defn.info.metaclass_type.type.has_base("enum.EnumMeta"): - defn.info.is_enum = True - if defn.type_vars: - self.fail("Enum class cannot be generic", defn) + if defn.info.metaclass_type and defn.info.metaclass_type.type.has_base("enum.EnumMeta"): + defn.info.is_enum = True + if defn.type_vars: + self.fail("Enum class cannot be generic", defn) # # Imports @@ -2277,10 +2407,9 @@ def report_missing_module_attribute( # Suggest alternatives, if any match is found. module = self.modules.get(import_id) if module: - if not self.options.implicit_reexport and source_id in module.names.keys(): + if source_id in module.names.keys() and not module.names[source_id].module_public: message = ( - 'Module "{}" does not explicitly export attribute "{}"' - "; implicit reexport disabled".format(import_id, source_id) + f'Module "{import_id}" does not explicitly export attribute "{source_id}"' ) else: alternatives = set(module.names.keys()).difference({source_id}) @@ -2693,30 +2822,32 @@ def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool: return False lvalue = s.lvalues[0] name = lvalue.name - internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple( - s.rvalue, name, self.is_func_scope() - ) - if internal_name is None: - return False - if isinstance(lvalue, MemberExpr): - self.fail("NamedTuple type as an attribute is not supported", lvalue) - return False - if internal_name != name: - self.fail( - 'First argument to namedtuple() should be "{}", not "{}"'.format( - name, internal_name - ), - s.rvalue, - code=codes.NAME_MATCH, + namespace = self.qualified_name(name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple( + s.rvalue, name, self.is_func_scope() ) + if internal_name is None: + return False + if isinstance(lvalue, MemberExpr): + self.fail("NamedTuple type as an attribute is not supported", lvalue) + return False + if internal_name != name: + self.fail( + 'First argument to namedtuple() should be "{}", not "{}"'.format( + name, internal_name + ), + s.rvalue, + code=codes.NAME_MATCH, + ) + return True + # Yes, it's a valid namedtuple, but defer if it is not ready. + if not info: + self.mark_incomplete(name, lvalue, becomes_typeinfo=True) + else: + self.setup_type_vars(info.defn, tvar_defs) + self.setup_alias_type_vars(info.defn) return True - # Yes, it's a valid namedtuple, but defer if it is not ready. - if not info: - self.mark_incomplete(name, lvalue, becomes_typeinfo=True) - else: - self.setup_type_vars(info.defn, tvar_defs) - self.setup_alias_type_vars(info.defn) - return True def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: """Check if s defines a typed dict.""" @@ -2730,22 +2861,24 @@ def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: return False lvalue = s.lvalues[0] name = lvalue.name - is_typed_dict, info, tvar_defs = self.typed_dict_analyzer.check_typeddict( - s.rvalue, name, self.is_func_scope() - ) - if not is_typed_dict: - return False - if isinstance(lvalue, MemberExpr): - self.fail("TypedDict type as attribute is not supported", lvalue) - return False - # Yes, it's a valid typed dict, but defer if it is not ready. - if not info: - self.mark_incomplete(name, lvalue, becomes_typeinfo=True) - else: - defn = info.defn - self.setup_type_vars(defn, tvar_defs) - self.setup_alias_type_vars(defn) - return True + namespace = self.qualified_name(name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + is_typed_dict, info, tvar_defs = self.typed_dict_analyzer.check_typeddict( + s.rvalue, name, self.is_func_scope() + ) + if not is_typed_dict: + return False + if isinstance(lvalue, MemberExpr): + self.fail("TypedDict type as attribute is not supported", lvalue) + return False + # Yes, it's a valid typed dict, but defer if it is not ready. + if not info: + self.mark_incomplete(name, lvalue, becomes_typeinfo=True) + else: + defn = info.defn + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) + return True def analyze_lvalues(self, s: AssignmentStmt) -> None: # We cannot use s.type, because analyze_simple_literal_type() will set it. @@ -2959,6 +3092,7 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) # Don't store not ready types (including placeholders). if analyzed is None or has_placeholder(analyzed): + self.defer(s) return s.type = analyzed if ( @@ -2980,7 +3114,13 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: self.fail("All protocol members must have explicitly declared types", s) # Set the type if the rvalue is a simple literal (even if the above error occurred). if len(s.lvalues) == 1 and isinstance(s.lvalues[0], RefExpr): - if s.lvalues[0].is_inferred_def: + ref_expr = s.lvalues[0] + safe_literal_inference = True + if self.type and isinstance(ref_expr, NameExpr) and len(self.type.mro) > 1: + # Check if there is a definition in supertype. If yes, we can't safely + # decide here what to infer: int or Literal[42]. + safe_literal_inference = self.type.mro[1].get(ref_expr.name) is None + if safe_literal_inference and ref_expr.is_inferred_def: s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) if s.type: # Store type into nodes. @@ -2998,7 +3138,6 @@ def is_annotated_protocol_member(self, s: AssignmentStmt) -> bool: def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Type | None: """Return builtins.int if rvalue is an int literal, etc. - If this is a 'Final' context, we return "Literal[...]" instead.""" if self.options.semantic_analysis_only or self.function_stack: # Skip this if we're only doing the semantic analysis pass. @@ -3149,11 +3288,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: res: Type | None = None if self.is_none_alias(rvalue): res = NoneType() - alias_tvars, depends_on, qualified_tvars = ( - [], - set(), - [], - ) # type: List[str], Set[str], List[str] + alias_tvars: list[str] = [] + depends_on: set[str] = set() + qualified_tvars: list[str] = [] else: tag = self.track_incomplete_refs() res, alias_tvars, depends_on, qualified_tvars = self.analyze_alias( @@ -3161,7 +3298,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: ) if not res: return False - if self.options.enable_recursive_aliases and not self.is_func_scope(): + if not self.options.disable_recursive_aliases and not self.is_func_scope(): # Only marking incomplete for top-level placeholders makes recursive aliases like # `A = Sequence[str | A]` valid here, similar to how we treat base classes in class # definitions, allowing `class str(Sequence[str]): ...` @@ -3241,6 +3378,12 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: current_node = existing.node if existing else alias_node assert isinstance(current_node, TypeAlias) self.disable_invalid_recursive_aliases(s, current_node) + if self.is_class_scope(): + assert self.type is not None + if self.type.is_protocol: + self.fail("Type aliases are prohibited in protocol bodies", s) + if not lvalue.name[0].isupper(): + self.note("Use variable annotation syntax to define protocol members", s) return True def disable_invalid_recursive_aliases( @@ -3292,7 +3435,7 @@ def analyze_lvalue( elif isinstance(lval, MemberExpr): self.analyze_member_lvalue(lval, explicit_type, is_final) if explicit_type and not self.is_self_member_ref(lval): - self.fail("Type cannot be declared in assignment to non-self " "attribute", lval) + self.fail("Type cannot be declared in assignment to non-self attribute", lval) elif isinstance(lval, IndexExpr): if explicit_type: self.fail("Unexpected type declaration", lval) @@ -3828,8 +3971,7 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: if len(call.args) > 1: self.fail("Only the first argument to TypeVarTuple has defined semantics", s) - if not self.options.enable_incomplete_features: - self.fail('"TypeVarTuple" is not supported by mypy yet', s) + if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s): return False name = self.extract_typevarlike_name(s, call) @@ -4862,6 +5004,7 @@ def visit_conditional_expr(self, expr: ConditionalExpr) -> None: def visit__promote_expr(self, expr: PromoteExpr) -> None: analyzed = self.anal_type(expr.type) if analyzed is not None: + assert isinstance(analyzed, ProperType), "Cannot use type aliases for promotions" expr.type = analyzed def visit_yield_expr(self, e: YieldExpr) -> None: @@ -5014,7 +5157,9 @@ class C: X = X # Initializer refers to outer scope Nested classes are an exception, since we want to support - arbitrary forward references in type annotations. + arbitrary forward references in type annotations. Also, we + allow forward references to type aliases to support recursive + types. """ # TODO: Forward reference to name imported in class body is not # caught. @@ -5025,7 +5170,7 @@ class C: node is None or self.is_textually_before_statement(node) or not self.is_defined_in_current_module(node.fullname) - or isinstance(node, TypeInfo) + or isinstance(node, (TypeInfo, TypeAlias)) or (isinstance(node, PlaceholderNode) and node.becomes_typeinfo) ) @@ -5107,6 +5252,11 @@ def lookup_qualified( if isinstance(typ, AnyType): # Allow access through Var with Any type without error. return self.implicit_symbol(sym, name, parts[i:], typ) + # This might be something like valid `P.args` or invalid `P.__bound__` access. + # Important note that `ParamSpecExpr` is also ignored in other places. + # See https://github.com/python/mypy/pull/13468 + if isinstance(node, ParamSpecExpr) and part in ("args", "kwargs"): + return None # Lookup through invalid node, such as variable or function nextsym = None if not nextsym or nextsym.module_hidden: @@ -5252,7 +5402,7 @@ def named_type_or_none(self, fullname: str, args: list[Type] | None = None) -> I return None node = sym.node if isinstance(node, TypeAlias): - assert isinstance(node.target, Instance) # type: ignore + assert isinstance(node.target, Instance) # type: ignore[misc] node = node.target.type assert isinstance(node, TypeInfo), node if args is not None: @@ -5659,7 +5809,7 @@ def process_placeholder(self, name: str, kind: str, ctx: Context) -> None: def cannot_resolve_name(self, name: str, kind: str, ctx: Context) -> None: self.fail(f'Cannot resolve {kind} "{name}" (possible cyclic definition)', ctx) - if self.options.enable_recursive_aliases and self.is_func_scope(): + if not self.options.disable_recursive_aliases and self.is_func_scope(): self.note("Recursive types are not allowed at function scope", ctx) def qualified_name(self, name: str) -> str: @@ -5850,7 +6000,7 @@ def in_checked_function(self) -> bool: current_index = len(self.function_stack) - 1 while current_index >= 0: current_func = self.function_stack[current_index] - if isinstance(current_func, FuncItem) and not isinstance(current_func, LambdaExpr): + if not isinstance(current_func, LambdaExpr): return not current_func.is_dynamic() # Special case, `lambda` inherits the "checked" state from its parent. @@ -5882,6 +6032,16 @@ def note(self, msg: str, ctx: Context, code: ErrorCode | None = None) -> None: return self.errors.report(ctx.get_line(), ctx.get_column(), msg, severity="note", code=code) + def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool: + if feature not in self.options.enable_incomplete_feature: + self.fail( + f'"{feature}" support is experimental,' + f" use --enable-incomplete-feature={feature} to enable", + ctx, + ) + return False + return True + def accept(self, node: Node) -> None: try: node.accept(self) diff --git a/mypy/semanal_classprop.py b/mypy/semanal_classprop.py index 654a29c38d08..5d21babcc597 100644 --- a/mypy/semanal_classprop.py +++ b/mypy/semanal_classprop.py @@ -22,7 +22,7 @@ Var, ) from mypy.options import Options -from mypy.types import Instance, Type +from mypy.types import Instance, ProperType # Hard coded type promotions (shared between all Python versions). # These add extra ad-hoc edges to the subtyping relation. For example, @@ -95,7 +95,7 @@ def calculate_class_abstract_status(typ: TypeInfo, is_stub_file: bool, errors: E # implement some methods. typ.abstract_attributes = sorted(abstract) if is_stub_file: - if typ.declared_metaclass and typ.declared_metaclass.type.fullname == "abc.ABCMeta": + if typ.declared_metaclass and typ.declared_metaclass.type.has_base("abc.ABCMeta"): return if typ.is_protocol: return @@ -155,7 +155,7 @@ def add_type_promotion( This includes things like 'int' being compatible with 'float'. """ defn = info.defn - promote_targets: list[Type] = [] + promote_targets: list[ProperType] = [] for decorator in defn.decorators: if isinstance(decorator, CallExpr): analyzed = decorator.analyzed @@ -165,6 +165,10 @@ def add_type_promotion( if not promote_targets: if defn.fullname in TYPE_PROMOTIONS: target_sym = module_names.get(TYPE_PROMOTIONS[defn.fullname]) + if defn.fullname == "builtins.bytearray" and options.disable_bytearray_promotion: + target_sym = None + elif defn.fullname == "builtins.memoryview" and options.disable_memoryview_promotion: + target_sym = None # With test stubs, the target may not exist. if target_sym: target_info = target_sym.node diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py index 406fd93139d1..9e3aeaa7fa4b 100644 --- a/mypy/semanal_main.py +++ b/mypy/semanal_main.py @@ -367,7 +367,11 @@ def check_type_arguments(graph: Graph, scc: list[str], errors: Errors) -> None: for module in scc: state = graph[module] assert state.tree - analyzer = TypeArgumentAnalyzer(errors, state.options, is_typeshed_file(state.path or "")) + analyzer = TypeArgumentAnalyzer( + errors, + state.options, + is_typeshed_file(state.options.abs_custom_typeshed_dir, state.path or ""), + ) with state.wrap_context(): with mypy.state.state.strict_optional_set(state.options.strict_optional): state.tree.accept(analyzer) @@ -381,7 +385,11 @@ def check_type_arguments_in_targets( This mirrors the logic in check_type_arguments() except that we process only some targets. This is used in fine grained incremental mode. """ - analyzer = TypeArgumentAnalyzer(errors, state.options, is_typeshed_file(state.path or "")) + analyzer = TypeArgumentAnalyzer( + errors, + state.options, + is_typeshed_file(state.options.abs_custom_typeshed_dir, state.path or ""), + ) with state.wrap_context(): with mypy.state.state.strict_optional_set(state.options.strict_optional): for target in targets: diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 4375602b5076..1727c18b6fd9 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -176,7 +176,7 @@ def check_namedtuple_classdef( # it would be inconsistent with type aliases. analyzed = self.api.anal_type( stmt.type, - allow_placeholder=self.options.enable_recursive_aliases + allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), ) if analyzed is None: @@ -321,11 +321,12 @@ def parse_namedtuple_args( ) -> None | (tuple[list[str], list[Type], list[Expression], str, list[TypeVarLikeType], bool]): """Parse a namedtuple() call into data needed to construct a type. - Returns a 5-tuple: + Returns a 6-tuple: - List of argument names - List of argument types - List of default values - First argument of namedtuple + - All typevars found in the field definition - Whether all types are ready. Return None if the definition didn't typecheck. @@ -442,7 +443,7 @@ def parse_namedtuple_fields_with_types( # We never allow recursive types at function scope. analyzed = self.api.anal_type( type, - allow_placeholder=self.options.enable_recursive_aliases + allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), ) # Workaround #4987 and avoid introducing a bogus UnboundType diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index b571ed538e09..b6fb64532e6e 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -203,7 +203,7 @@ def check_newtype_args( self.api.anal_type( unanalyzed_type, report_invalid_types=False, - allow_placeholder=self.options.enable_recursive_aliases + allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), ) ) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index d9ded032591b..63f4f5516f79 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -82,6 +82,10 @@ def fail( def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None: raise NotImplementedError + @abstractmethod + def incomplete_feature_enabled(self, feature: str, ctx: Context) -> bool: + raise NotImplementedError + @abstractmethod def record_incomplete_ref(self) -> None: raise NotImplementedError diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index f988014cdd02..161775ce8fd9 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -46,7 +46,7 @@ def __init__(self, errors: Errors, options: Options, is_typeshed_file: bool) -> self.seen_aliases: set[TypeAliasType] = set() def visit_mypy_file(self, o: MypyFile) -> None: - self.errors.set_file(o.path, o.fullname, scope=self.scope) + self.errors.set_file(o.path, o.fullname, scope=self.scope, options=self.options) with self.scope.module_scope(o.fullname): super().visit_mypy_file(o) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 77e83e53f686..fd6b1bbd2bbf 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -4,7 +4,7 @@ from typing_extensions import Final -from mypy import errorcodes as codes +from mypy import errorcodes as codes, message_registry from mypy.errorcodes import ErrorCode from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type from mypy.messages import MessageBuilder @@ -79,6 +79,9 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N self.api.accept(base_expr) if base_expr.fullname in TPDICT_NAMES or self.is_typeddict(base_expr): possible = True + if isinstance(base_expr.node, TypeInfo) and base_expr.node.is_final: + err = message_registry.CANNOT_INHERIT_FROM_FINAL + self.fail(err.format(base_expr.node.name).value, defn, code=err.code) if not possible: return False, None existing_info = None @@ -215,7 +218,7 @@ def analyze_base_args(self, base: IndexExpr, ctx: Context) -> list[Type] | None: analyzed = self.api.anal_type( type, allow_required=True, - allow_placeholder=self.options.enable_recursive_aliases + allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), ) if analyzed is None: @@ -286,7 +289,7 @@ def analyze_typeddict_classdef_fields( analyzed = self.api.anal_type( stmt.type, allow_required=True, - allow_placeholder=self.options.enable_recursive_aliases + allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), ) if analyzed is None: @@ -481,7 +484,7 @@ def parse_typeddict_fields_with_types( analyzed = self.api.anal_type( type, allow_required=True, - allow_placeholder=self.options.enable_recursive_aliases + allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), ) if analyzed is None: diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index e913188df02f..41a79db480c9 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -52,13 +52,15 @@ class level -- these are handled at attribute level (say, 'mod.Cls.method' from __future__ import annotations -from typing import Sequence, Tuple +from typing import Sequence, Tuple, cast from typing_extensions import TypeAlias as _TypeAlias +from mypy.expandtype import expand_type from mypy.nodes import ( UNBOUND_IMPORTED, Decorator, FuncBase, + FuncDef, FuncItem, MypyFile, OverloadedFuncDef, @@ -68,6 +70,7 @@ class level -- these are handled at attribute level (say, 'mod.Cls.method' TypeAlias, TypeInfo, TypeVarExpr, + TypeVarTupleExpr, Var, ) from mypy.types import ( @@ -87,6 +90,8 @@ class level -- these are handled at attribute level (say, 'mod.Cls.method' TypeAliasType, TypedDictType, TypeType, + TypeVarId, + TypeVarLikeType, TypeVarTupleType, TypeVarType, TypeVisitor, @@ -189,6 +194,8 @@ def snapshot_symbol_table(name_prefix: str, table: SymbolTable) -> dict[str, Sna ) elif isinstance(node, ParamSpecExpr): result[name] = ("ParamSpec", node.variance, snapshot_type(node.upper_bound)) + elif isinstance(node, TypeVarTupleExpr): + result[name] = ("TypeVarTuple", node.variance, snapshot_type(node.upper_bound)) else: assert symbol.kind != UNBOUND_IMPORTED if node and get_prefix(node.fullname) != name_prefix: @@ -211,6 +218,12 @@ def snapshot_definition(node: SymbolNode | None, common: tuple[object, ...]) -> signature = snapshot_type(node.type) else: signature = snapshot_untyped_signature(node) + impl: FuncDef | None = None + if isinstance(node, FuncDef): + impl = node + elif isinstance(node, OverloadedFuncDef) and node.impl: + impl = node.impl.func if isinstance(node.impl, Decorator) else node.impl + is_trivial_body = impl.is_trivial_body if impl else False return ( "Func", common, @@ -219,6 +232,7 @@ def snapshot_definition(node: SymbolNode | None, common: tuple[object, ...]) -> node.is_class, node.is_static, signature, + is_trivial_body, ) elif isinstance(node, Var): return ("Var", common, snapshot_optional_type(node.type), node.is_final) @@ -385,7 +399,8 @@ def visit_parameters(self, typ: Parameters) -> SnapshotItem: ) def visit_callable_type(self, typ: CallableType) -> SnapshotItem: - # FIX generics + if typ.is_generic(): + typ = self.normalize_callable_variables(typ) return ( "CallableType", snapshot_types(typ.arg_types), @@ -394,8 +409,26 @@ def visit_callable_type(self, typ: CallableType) -> SnapshotItem: tuple(typ.arg_kinds), typ.is_type_obj(), typ.is_ellipsis_args, + snapshot_types(typ.variables), ) + def normalize_callable_variables(self, typ: CallableType) -> CallableType: + """Normalize all type variable ids to run from -1 to -len(variables).""" + tvs = [] + tvmap: dict[TypeVarId, Type] = {} + for i, v in enumerate(typ.variables): + tid = TypeVarId(-1 - i) + if isinstance(v, TypeVarType): + tv: TypeVarLikeType = v.copy_modified(id=tid) + elif isinstance(v, TypeVarTupleType): + tv = v.copy_modified(id=tid) + else: + assert isinstance(v, ParamSpecType) + tv = v.copy_modified(id=tid) + tvs.append(tv) + tvmap[v.id] = tv + return cast(CallableType, expand_type(typ, tvmap)).copy_modified(variables=tvs) + def visit_tuple_type(self, typ: TupleType) -> SnapshotItem: return ("TupleType", snapshot_types(typ.items)) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 7cb4aeda7534..45d7947641da 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -81,7 +81,8 @@ class 'mod.Cls'. This can also refer to an attribute inherited from a from __future__ import annotations -from typing import DefaultDict, List +from collections import defaultdict +from typing import List from mypy.nodes import ( GDEF, @@ -220,7 +221,7 @@ def __init__( self, type_map: dict[Expression, Type], python_version: tuple[int, int], - alias_deps: DefaultDict[str, set[str]], + alias_deps: defaultdict[str, set[str]], options: Options | None = None, ) -> None: self.scope = Scope() @@ -968,6 +969,9 @@ def visit_instance(self, typ: Instance) -> list[str]: triggers.extend(self.get_type_triggers(arg)) if typ.last_known_value: triggers.extend(self.get_type_triggers(typ.last_known_value)) + if typ.extra_attrs and typ.extra_attrs.mod_name: + # Module as type effectively depends on all module attributes, use wildcard. + triggers.append(make_wildcard_trigger(typ.extra_attrs.mod_name)) return triggers def visit_type_alias_type(self, typ: TypeAliasType) -> list[str]: diff --git a/mypy/server/objgraph.py b/mypy/server/objgraph.py index f15d503f0f16..89a086b8a0ab 100644 --- a/mypy/server/objgraph.py +++ b/mypy/server/objgraph.py @@ -64,11 +64,11 @@ def get_edges(o: object) -> Iterator[tuple[object, object]]: # in closures and self pointers to other objects if hasattr(e, "__closure__"): - yield (s, "__closure__"), e.__closure__ # type: ignore + yield (s, "__closure__"), e.__closure__ if hasattr(e, "__self__"): - se = e.__self__ # type: ignore + se = e.__self__ if se is not o and se is not type(o) and hasattr(s, "__self__"): - yield s.__self__, se # type: ignore + yield s.__self__, se else: if not type(e) in TYPE_BLACKLIST: yield s, e diff --git a/mypy/server/update.py b/mypy/server/update.py index ed059259c7a6..686068a4aad0 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -115,6 +115,7 @@ from __future__ import annotations import os +import re import sys import time from typing import Callable, NamedTuple, Sequence, Union @@ -182,7 +183,7 @@ def __init__(self, result: BuildResult) -> None: # Merge in any root dependencies that may not have been loaded merge_dependencies(manager.load_fine_grained_deps(FAKE_ROOT_MODULE), self.deps) self.previous_targets_with_errors = manager.errors.targets() - self.previous_messages = result.errors[:] + self.previous_messages: list[str] = result.errors[:] # Module, if any, that had blocking errors in the last run as (id, path) tuple. self.blocking_error: tuple[str, str] | None = None # Module that we haven't processed yet but that are known to be stale. @@ -290,6 +291,7 @@ def update( messages = self.manager.errors.new_messages() break + messages = sort_messages_preserving_file_order(messages, self.previous_messages) self.previous_messages = messages[:] return messages @@ -651,6 +653,7 @@ def restore(ids: list[str]) -> None: state.type_checker().reset() state.type_check_first_pass() state.type_check_second_pass() + state.detect_partially_defined_vars(state.type_map()) t2 = time.time() state.finish_passes() t3 = time.time() @@ -962,9 +965,10 @@ def key(node: FineGrainedDeferredNode) -> int: nodes = sorted(nodeset, key=key) - options = graph[module_id].options + state = graph[module_id] + options = state.options manager.errors.set_file_ignored_lines( - file_node.path, file_node.ignored_lines, options.ignore_errors + file_node.path, file_node.ignored_lines, options.ignore_errors or state.ignore_all ) targets = set() @@ -1258,3 +1262,61 @@ def refresh_suppressed_submodules( state.suppressed.append(submodule) state.suppressed_set.add(submodule) return messages + + +def extract_fnam_from_message(message: str) -> str | None: + m = re.match(r"([^:]+):[0-9]+: (error|note): ", message) + if m: + return m.group(1) + return None + + +def extract_possible_fnam_from_message(message: str) -> str: + # This may return non-path things if there is some random colon on the line + return message.split(":", 1)[0] + + +def sort_messages_preserving_file_order( + messages: list[str], prev_messages: list[str] +) -> list[str]: + """Sort messages so that the order of files is preserved. + + An update generates messages so that the files can be in a fairly + arbitrary order. Preserve the order of files to avoid messages + getting reshuffled continuously. If there are messages in + additional files, sort them towards the end. + """ + # Calculate file order from the previous messages + n = 0 + order = {} + for msg in prev_messages: + fnam = extract_fnam_from_message(msg) + if fnam and fnam not in order: + order[fnam] = n + n += 1 + + # Related messages must be sorted as a group of successive lines + groups = [] + i = 0 + while i < len(messages): + msg = messages[i] + maybe_fnam = extract_possible_fnam_from_message(msg) + group = [msg] + if maybe_fnam in order: + # This looks like a file name. Collect all lines related to this message. + while ( + i + 1 < len(messages) + and extract_possible_fnam_from_message(messages[i + 1]) not in order + and extract_fnam_from_message(messages[i + 1]) is None + and not messages[i + 1].startswith("mypy: ") + ): + i += 1 + group.append(messages[i]) + groups.append((order.get(maybe_fnam, n), group)) + i += 1 + + groups = sorted(groups, key=lambda g: g[0]) + result = [] + for key, group in groups: + result.extend(group) + return result diff --git a/mypy/stats.py b/mypy/stats.py index f68edd7b9c04..af6c5fc14a50 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -3,7 +3,6 @@ from __future__ import annotations import os -import typing from collections import Counter from contextlib import contextmanager from typing import Iterator, cast @@ -102,7 +101,7 @@ def __init__( self.line_map: dict[int, int] = {} - self.type_of_any_counter: typing.Counter[int] = Counter() + self.type_of_any_counter: Counter[int] = Counter() self.any_line_map: dict[int, list[AnyType]] = {} # For each scope (top level/function), whether the scope was type checked diff --git a/mypy/strconv.py b/mypy/strconv.py index 1acf7699316c..9b369618b88e 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -276,6 +276,8 @@ def visit_del_stmt(self, o: mypy.nodes.DelStmt) -> str: def visit_try_stmt(self, o: mypy.nodes.TryStmt) -> str: a: list[Any] = [o.body] + if o.is_star: + a.append("*") for i in range(len(o.vars)): a.append(o.types[i]) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 243db68f7a80..fbae9ebaa252 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -48,7 +48,7 @@ import sys import traceback from collections import defaultdict -from typing import Dict, Iterable, List, Mapping, Optional, cast +from typing import Iterable, List, Mapping, cast from typing_extensions import Final import mypy.build @@ -104,7 +104,13 @@ ) from mypy.options import Options as MypyOptions from mypy.stubdoc import Sig, find_unique_signatures, parse_all_signatures -from mypy.stubgenc import generate_stub_for_c_module +from mypy.stubgenc import ( + DocstringSignatureGenerator, + ExternalSignatureGenerator, + FallbackSignatureGenerator, + SignatureGenerator, + generate_stub_for_c_module, +) from mypy.stubutil import ( CantImport, common_dir_prefix, @@ -183,6 +189,22 @@ "__iter__", } +# These magic methods always return the same type. +KNOWN_MAGIC_METHODS_RETURN_TYPES: Final = { + "__len__": "int", + "__length_hint__": "int", + "__init__": "None", + "__del__": "None", + "__bool__": "bool", + "__bytes__": "bytes", + "__format__": "str", + "__contains__": "bool", + "__complex__": "complex", + "__int__": "int", + "__float__": "float", + "__index__": "int", +} + class Options: """Represents stubgen options. @@ -643,7 +665,7 @@ def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: self.visit_func_def(item.func, is_abstract=is_abstract, is_overload=is_overload) if is_overload: overload_chain = True - elif overload_chain and is_overload: + elif is_overload: self.visit_func_def(item.func, is_abstract=is_abstract, is_overload=is_overload) else: # skip the overload implementation and clear the decorator we just processed @@ -725,12 +747,12 @@ def visit_func_def( retname = None # implicit Any else: retname = self.print_annotation(o.unanalyzed_type.ret_type) - elif isinstance(o, FuncDef) and ( - o.abstract_status == IS_ABSTRACT or o.name in METHODS_WITH_RETURN_VALUE - ): + elif o.abstract_status == IS_ABSTRACT or o.name in METHODS_WITH_RETURN_VALUE: # Always assume abstract methods return Any unless explicitly annotated. Also # some dunder methods should not have a None return type. retname = None # implicit Any + elif o.name in KNOWN_MAGIC_METHODS_RETURN_TYPES: + retname = KNOWN_MAGIC_METHODS_RETURN_TYPES[o.name] elif has_yield_expression(o): self.add_abc_import("Generator") yield_name = "None" @@ -985,7 +1007,7 @@ def visit_assignment_stmt(self, o: AssignmentStmt) -> None: continue if isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): items = lvalue.items - if isinstance(o.unanalyzed_type, TupleType): # type: ignore + if isinstance(o.unanalyzed_type, TupleType): # type: ignore[misc] annotations: Iterable[Type | None] = o.unanalyzed_type.items else: annotations = [None] * len(items) @@ -1628,6 +1650,18 @@ def generate_stub_from_ast( file.write("".join(gen.output())) +def get_sig_generators(options: Options) -> List[SignatureGenerator]: + sig_generators: List[SignatureGenerator] = [ + DocstringSignatureGenerator(), + FallbackSignatureGenerator(), + ] + if options.doc_dir: + # Collect info from docs (if given). Always check these first. + sigs, class_sigs = collect_docs_signatures(options.doc_dir) + sig_generators.insert(0, ExternalSignatureGenerator(sigs, class_sigs)) + return sig_generators + + def collect_docs_signatures(doc_dir: str) -> tuple[dict[str, str], dict[str, str]]: """Gather all function and class signatures in the docs. @@ -1650,12 +1684,7 @@ def generate_stubs(options: Options) -> None: """Main entry point for the program.""" mypy_opts = mypy_options(options) py_modules, c_modules = collect_build_targets(options, mypy_opts) - - # Collect info from docs (if given): - sigs = class_sigs = None # type: Optional[Dict[str, str]] - if options.doc_dir: - sigs, class_sigs = collect_docs_signatures(options.doc_dir) - + sig_generators = get_sig_generators(options) # Use parsed sources to generate stubs for Python modules. generate_asts_for_modules(py_modules, options.parse_only, mypy_opts, options.verbose) files = [] @@ -1682,7 +1711,7 @@ def generate_stubs(options: Options) -> None: target = os.path.join(options.output_dir, target) files.append(target) with generate_guarded(mod.module, target, options.ignore_errors, options.verbose): - generate_stub_for_c_module(mod.module, target, sigs=sigs, class_sigs=class_sigs) + generate_stub_for_c_module(mod.module, target, sig_generators=sig_generators) num_modules = len(py_modules) + len(c_modules) if not options.quiet and num_modules > 0: print("Processed %d modules" % num_modules) @@ -1733,9 +1762,7 @@ def parse_options(args: list[str]) -> Options: parser.add_argument( "--export-less", action="store_true", - help=( - "don't implicitly export all names imported from other modules " "in the same package" - ), + help="don't implicitly export all names imported from other modules in the same package", ) parser.add_argument("-v", "--verbose", action="store_true", help="show more verbose messages") parser.add_argument("-q", "--quiet", action="store_true", help="show fewer messages") diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 6b3f9d47b34a..add33e66cee3 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -10,8 +10,9 @@ import inspect import os.path import re +from abc import abstractmethod from types import ModuleType -from typing import Any, Mapping +from typing import Any, Iterable, Mapping from typing_extensions import Final from mypy.moduleinspect import is_c_module @@ -40,16 +41,119 @@ ) +class SignatureGenerator: + """Abstract base class for extracting a list of FunctionSigs for each function.""" + + @abstractmethod + def get_function_sig( + self, func: object, module_name: str, name: str + ) -> list[FunctionSig] | None: + pass + + @abstractmethod + def get_method_sig( + self, func: object, module_name: str, class_name: str, name: str, self_var: str + ) -> list[FunctionSig] | None: + pass + + +class ExternalSignatureGenerator(SignatureGenerator): + def __init__( + self, func_sigs: dict[str, str] | None = None, class_sigs: dict[str, str] | None = None + ): + """ + Takes a mapping of function/method names to signatures and class name to + class signatures (usually corresponds to __init__). + """ + self.func_sigs = func_sigs or {} + self.class_sigs = class_sigs or {} + + def get_function_sig( + self, func: object, module_name: str, name: str + ) -> list[FunctionSig] | None: + if name in self.func_sigs: + return [ + FunctionSig( + name=name, + args=infer_arg_sig_from_anon_docstring(self.func_sigs[name]), + ret_type="Any", + ) + ] + else: + return None + + def get_method_sig( + self, func: object, module_name: str, class_name: str, name: str, self_var: str + ) -> list[FunctionSig] | None: + if ( + name in ("__new__", "__init__") + and name not in self.func_sigs + and class_name in self.class_sigs + ): + return [ + FunctionSig( + name=name, + args=infer_arg_sig_from_anon_docstring(self.class_sigs[class_name]), + ret_type="None" if name == "__init__" else "Any", + ) + ] + return self.get_function_sig(func, module_name, name) + + +class DocstringSignatureGenerator(SignatureGenerator): + def get_function_sig( + self, func: object, module_name: str, name: str + ) -> list[FunctionSig] | None: + docstr = getattr(func, "__doc__", None) + inferred = infer_sig_from_docstring(docstr, name) + if inferred: + assert docstr is not None + if is_pybind11_overloaded_function_docstring(docstr, name): + # Remove pybind11 umbrella (*args, **kwargs) for overloaded functions + del inferred[-1] + return inferred + + def get_method_sig( + self, func: object, module_name: str, class_name: str, name: str, self_var: str + ) -> list[FunctionSig] | None: + return self.get_function_sig(func, module_name, name) + + +class FallbackSignatureGenerator(SignatureGenerator): + def get_function_sig( + self, func: object, module_name: str, name: str + ) -> list[FunctionSig] | None: + return [ + FunctionSig( + name=name, + args=infer_arg_sig_from_anon_docstring("(*args, **kwargs)"), + ret_type="Any", + ) + ] + + def get_method_sig( + self, func: object, module_name: str, class_name: str, name: str, self_var: str + ) -> list[FunctionSig] | None: + return [ + FunctionSig( + name=name, + args=infer_method_sig(name, self_var), + ret_type="None" if name == "__init__" else "Any", + ) + ] + + def generate_stub_for_c_module( - module_name: str, - target: str, - sigs: dict[str, str] | None = None, - class_sigs: dict[str, str] | None = None, + module_name: str, target: str, sig_generators: Iterable[SignatureGenerator] ) -> None: """Generate stub for C module. - This combines simple runtime introspection (looking for docstrings and attributes - with simple builtin types) and signatures inferred from .rst documentation (if given). + Signature generators are called in order until a list of signatures is returned. The order + is: + - signatures inferred from .rst documentation (if given) + - simple runtime introspection (looking for docstrings and attributes + with simple builtin types) + - fallback based special method names or "(*args, **kwargs)" If directory for target doesn't exist it will be created. Existing stub will be overwritten. @@ -65,7 +169,9 @@ def generate_stub_for_c_module( items = sorted(module.__dict__.items(), key=lambda x: x[0]) for name, obj in items: if is_c_function(obj): - generate_c_function_stub(module, name, obj, functions, imports=imports, sigs=sigs) + generate_c_function_stub( + module, name, obj, functions, imports=imports, sig_generators=sig_generators + ) done.add(name) types: list[str] = [] for name, obj in items: @@ -73,7 +179,7 @@ def generate_stub_for_c_module( continue if is_c_type(obj): generate_c_type_stub( - module, name, obj, types, imports=imports, sigs=sigs, class_sigs=class_sigs + module, name, obj, types, imports=imports, sig_generators=sig_generators ) done.add(name) variables = [] @@ -153,10 +259,9 @@ def generate_c_function_stub( obj: object, output: list[str], imports: list[str], + sig_generators: Iterable[SignatureGenerator], self_var: str | None = None, - sigs: dict[str, str] | None = None, class_name: str | None = None, - class_sigs: dict[str, str] | None = None, ) -> None: """Generate stub for a single function or method. @@ -165,60 +270,38 @@ def generate_c_function_stub( The 'class_name' is used to find signature of __init__ or __new__ in 'class_sigs'. """ - if sigs is None: - sigs = {} - if class_sigs is None: - class_sigs = {} - - ret_type = "None" if name == "__init__" and class_name else "Any" - - if ( - name in ("__new__", "__init__") - and name not in sigs - and class_name - and class_name in class_sigs - ): - inferred: list[FunctionSig] | None = [ - FunctionSig( - name=name, - args=infer_arg_sig_from_anon_docstring(class_sigs[class_name]), - ret_type=ret_type, - ) - ] + inferred: list[FunctionSig] | None = None + if class_name: + # method: + assert self_var is not None, "self_var should be provided for methods" + for sig_gen in sig_generators: + inferred = sig_gen.get_method_sig(obj, module.__name__, class_name, name, self_var) + if inferred: + # add self/cls var, if not present + for sig in inferred: + if not sig.args or sig.args[0].name != self_var: + sig.args.insert(0, ArgSig(name=self_var)) + break else: - docstr = getattr(obj, "__doc__", None) - inferred = infer_sig_from_docstring(docstr, name) - if inferred: - assert docstr is not None - if is_pybind11_overloaded_function_docstring(docstr, name): - # Remove pybind11 umbrella (*args, **kwargs) for overloaded functions - del inferred[-1] - if not inferred: - if class_name and name not in sigs: - inferred = [ - FunctionSig(name, args=infer_method_sig(name, self_var), ret_type=ret_type) - ] - else: - inferred = [ - FunctionSig( - name=name, - args=infer_arg_sig_from_anon_docstring( - sigs.get(name, "(*args, **kwargs)") - ), - ret_type=ret_type, - ) - ] - elif class_name and self_var: - args = inferred[0].args - if not args or args[0].name != self_var: - args.insert(0, ArgSig(name=self_var)) + # function: + for sig_gen in sig_generators: + inferred = sig_gen.get_function_sig(obj, module.__name__, name) + if inferred: + break + + if not inferred: + raise ValueError( + "No signature was found. This should never happen " + "if FallbackSignatureGenerator is provided" + ) + is_classmethod = self_var == "cls" is_overloaded = len(inferred) > 1 if inferred else False if is_overloaded: imports.append("from typing import overload") if inferred: for signature in inferred: - sig = [] + args: list[str] = [] for arg in signature.args: if arg.name == self_var: arg_def = self_var @@ -233,14 +316,16 @@ def generate_c_function_stub( if arg.default: arg_def += " = ..." - sig.append(arg_def) + args.append(arg_def) if is_overloaded: output.append("@overload") + if is_classmethod: + output.append("@classmethod") output.append( "def {function}({args}) -> {ret}: ...".format( function=name, - args=", ".join(sig), + args=", ".join(args), ret=strip_or_import(signature.ret_type, module, imports), ) ) @@ -338,8 +423,7 @@ def generate_c_type_stub( obj: type, output: list[str], imports: list[str], - sigs: dict[str, str] | None = None, - class_sigs: dict[str, str] | None = None, + sig_generators: Iterable[SignatureGenerator], ) -> None: """Generate stub for a single class using runtime introspection. @@ -369,7 +453,6 @@ def generate_c_type_stub( continue attr = "__init__" if is_c_classmethod(value): - methods.append("@classmethod") self_var = "cls" else: self_var = "self" @@ -380,9 +463,8 @@ def generate_c_type_stub( methods, imports=imports, self_var=self_var, - sigs=sigs, class_name=class_name, - class_sigs=class_sigs, + sig_generators=sig_generators, ) elif is_c_property(value): done.add(attr) @@ -398,7 +480,7 @@ def generate_c_type_stub( ) elif is_c_type(value): generate_c_type_stub( - module, attr, value, types, imports=imports, sigs=sigs, class_sigs=class_sigs + module, attr, value, types, imports=imports, sig_generators=sig_generators ) done.add(attr) diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py index ef025e1caa0f..b8dea5d0046b 100644 --- a/mypy/stubinfo.py +++ b/mypy/stubinfo.py @@ -5,6 +5,14 @@ def is_legacy_bundled_package(prefix: str) -> bool: return prefix in legacy_bundled_packages +def approved_stub_package_exists(prefix: str) -> bool: + return is_legacy_bundled_package(prefix) or prefix in non_bundled_packages + + +def stub_package_name(prefix: str) -> str: + return legacy_bundled_packages.get(prefix) or non_bundled_packages[prefix] + + # Stubs for these third-party packages used to be shipped with mypy. # # Map package name to PyPI stub distribution name. @@ -64,3 +72,98 @@ def is_legacy_bundled_package(prefix: str) -> bool: "waitress": "types-waitress", "yaml": "types-PyYAML", } + +# Map package name to PyPI stub distribution name from typeshed. +# Stubs for these packages were never bundled with mypy. Don't +# include packages that have a release that includes PEP 561 type +# information. +# +# Package name can have one or two components ('a' or 'a.b'). +# +# Note that these packages are omitted for now: +# sqlalchemy: It's unclear which stub package to suggest. There's also +# a mypy plugin available. +non_bundled_packages = { + "MySQLdb": "types-mysqlclient", + "PIL": "types-Pillow", + "PyInstaller": "types-pyinstaller", + "annoy": "types-annoy", + "appdirs": "types-appdirs", + "aws_xray_sdk": "types-aws-xray-sdk", + "babel": "types-babel", + "backports.ssl_match_hostname": "types-backports.ssl_match_hostname", + "braintree": "types-braintree", + "bs4": "types-beautifulsoup4", + "bugbear": "types-flake8-bugbear", + "caldav": "types-caldav", + "cffi": "types-cffi", + "chevron": "types-chevron", + "colorama": "types-colorama", + "commonmark": "types-commonmark", + "cryptography": "types-cryptography", + "d3dshot": "types-D3DShot", + "dj_database_url": "types-dj-database-url", + "docopt": "types-docopt", + "editdistance": "types-editdistance", + "entrypoints": "types-entrypoints", + "farmhash": "types-pyfarmhash", + "flake8_2020": "types-flake8-2020", + "flake8_builtins": "types-flake8-builtins", + "flake8_docstrings": "types-flake8-docstrings", + "flake8_plugin_utils": "types-flake8-plugin-utils", + "flake8_rst_docstrings": "types-flake8-rst-docstrings", + "flake8_simplify": "types-flake8-simplify", + "flake8_typing_imports": "types-flake8-typing-imports", + "flask_cors": "types-Flask-Cors", + "flask_sqlalchemy": "types-Flask-SQLAlchemy", + "fpdf": "types-fpdf2", + "gdb": "types-gdb", + "google.cloud": "types-google-cloud-ndb", + "hdbcli": "types-hdbcli", + "html5lib": "types-html5lib", + "httplib2": "types-httplib2", + "humanfriendly": "types-humanfriendly", + "invoke": "types-invoke", + "jack": "types-JACK-Client", + "jmespath": "types-jmespath", + "jose": "types-python-jose", + "jsonschema": "types-jsonschema", + "keyboard": "types-keyboard", + "ldap3": "types-ldap3", + "nmap": "types-python-nmap", + "oauthlib": "types-oauthlib", + "openpyxl": "types-openpyxl", + "opentracing": "types-opentracing", + "parsimonious": "types-parsimonious", + "passlib": "types-passlib", + "passpy": "types-passpy", + "pep8ext_naming": "types-pep8-naming", + "playsound": "types-playsound", + "prettytable": "types-prettytable", + "psutil": "types-psutil", + "psycopg2": "types-psycopg2", + "pyaudio": "types-pyaudio", + "pyautogui": "types-PyAutoGUI", + "pyflakes": "types-pyflakes", + "pygments": "types-Pygments", + "pyi_splash": "types-pyinstaller", + "pynput": "types-pynput", + "pysftp": "types-pysftp", + "pytest_lazyfixture": "types-pytest-lazy-fixture", + "regex": "types-regex", + "send2trash": "types-Send2Trash", + "slumber": "types-slumber", + "stdlib_list": "types-stdlib-list", + "stripe": "types-stripe", + "toposort": "types-toposort", + "tqdm": "types-tqdm", + "tree_sitter": "types-tree-sitter", + "tree_sitter_languages": "types-tree-sitter-languages", + "ttkthemes": "types-ttkthemes", + "urllib3": "types-urllib3", + "vobject": "types-vobject", + "whatthepatch": "types-whatthepatch", + "xmltodict": "types-xmltodict", + "xxhash": "types-xxhash", + "zxcvbn": "types-zxcvbn", +} diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 378f61471437..87ccbd3176df 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -136,7 +136,7 @@ def get_description(self, concise: bool = False) -> str: if not isinstance(self.runtime_object, Missing): try: runtime_line = inspect.getsourcelines(self.runtime_object)[1] - except (OSError, TypeError): + except (OSError, TypeError, SyntaxError): pass try: runtime_file = inspect.getsourcefile(self.runtime_object) @@ -315,7 +315,7 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: except Exception: return False if obj_mod is not None: - return obj_mod == r.__name__ + return bool(obj_mod == r.__name__) return not isinstance(obj, types.ModuleType) runtime_public_contents = ( @@ -349,20 +349,12 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: yield from verify(stub_entry, runtime_entry, object_path + [entry]) -@verify.register(nodes.TypeInfo) -def verify_typeinfo( - stub: nodes.TypeInfo, runtime: MaybeMissing[type[Any]], object_path: list[str] +def _verify_final( + stub: nodes.TypeInfo, runtime: type[Any], object_path: list[str] ) -> Iterator[Error]: - if isinstance(runtime, Missing): - yield Error(object_path, "is not present at runtime", stub, runtime, stub_desc=repr(stub)) - return - if not isinstance(runtime, type): - yield Error(object_path, "is not a type", stub, runtime, stub_desc=repr(stub)) - return - try: - class SubClass(runtime): # type: ignore + class SubClass(runtime): # type: ignore[misc,valid-type] pass except TypeError: @@ -380,6 +372,59 @@ class SubClass(runtime): # type: ignore # Examples: ctypes.Array, ctypes._SimpleCData pass + +def _verify_metaclass( + stub: nodes.TypeInfo, runtime: type[Any], object_path: list[str] +) -> Iterator[Error]: + # We exclude protocols, because of how complex their implementation is in different versions of + # python. Enums are also hard, ignoring. + # TODO: check that metaclasses are identical? + if not stub.is_protocol and not stub.is_enum: + runtime_metaclass = type(runtime) + if runtime_metaclass is not type and stub.metaclass_type is None: + # This means that runtime has a custom metaclass, but a stub does not. + yield Error( + object_path, + "is inconsistent, metaclass differs", + stub, + runtime, + stub_desc="N/A", + runtime_desc=f"{runtime_metaclass}", + ) + elif ( + runtime_metaclass is type + and stub.metaclass_type is not None + # We ignore extra `ABCMeta` metaclass on stubs, this might be typing hack. + # We also ignore `builtins.type` metaclass as an implementation detail in mypy. + and not mypy.types.is_named_instance( + stub.metaclass_type, ("abc.ABCMeta", "builtins.type") + ) + ): + # This means that our stub has a metaclass that is not present at runtime. + yield Error( + object_path, + "metaclass mismatch", + stub, + runtime, + stub_desc=f"{stub.metaclass_type.type.fullname}", + runtime_desc="N/A", + ) + + +@verify.register(nodes.TypeInfo) +def verify_typeinfo( + stub: nodes.TypeInfo, runtime: MaybeMissing[type[Any]], object_path: list[str] +) -> Iterator[Error]: + if isinstance(runtime, Missing): + yield Error(object_path, "is not present at runtime", stub, runtime, stub_desc=repr(stub)) + return + if not isinstance(runtime, type): + yield Error(object_path, "is not a type", stub, runtime, stub_desc=repr(stub)) + return + + yield from _verify_final(stub, runtime, object_path) + yield from _verify_metaclass(stub, runtime, object_path) + # Check everything already defined on the stub class itself (i.e. not inherited) to_check = set(stub.names) # Check all public things on the runtime class @@ -569,7 +614,7 @@ def get_type(arg: Any) -> str | None: def has_default(arg: Any) -> bool: if isinstance(arg, inspect.Parameter): - return arg.default != inspect.Parameter.empty + return bool(arg.default != inspect.Parameter.empty) if isinstance(arg, nodes.Argument): return arg.kind.is_optional() raise AssertionError @@ -825,16 +870,8 @@ def verify_funcitem( return if isinstance(stub, nodes.FuncDef): - stub_abstract = stub.abstract_status == nodes.IS_ABSTRACT - runtime_abstract = getattr(runtime, "__isabstractmethod__", False) - # The opposite can exist: some implementations omit `@abstractmethod` decorators - if runtime_abstract and not stub_abstract: - yield Error( - object_path, - "is inconsistent, runtime method is abstract but stub is not", - stub, - runtime, - ) + for error_text in _verify_abstract_status(stub, runtime): + yield Error(object_path, error_text, stub, runtime) for message in _verify_static_class_methods(stub, runtime, object_path): yield Error(object_path, "is inconsistent, " + message, stub, runtime) @@ -1021,6 +1058,15 @@ def _verify_readonly_property(stub: nodes.Decorator, runtime: Any) -> Iterator[s yield "is inconsistent, cannot reconcile @property on stub with runtime object" +def _verify_abstract_status(stub: nodes.FuncDef, runtime: Any) -> Iterator[str]: + stub_abstract = stub.abstract_status == nodes.IS_ABSTRACT + runtime_abstract = getattr(runtime, "__isabstractmethod__", False) + # The opposite can exist: some implementations omit `@abstractmethod` decorators + if runtime_abstract and not stub_abstract: + item_type = "property" if stub.is_property else "method" + yield f"is inconsistent, runtime {item_type} is abstract but stub is not" + + def _resolve_funcitem_from_decorator(dec: nodes.OverloadPart) -> nodes.FuncItem | None: """Returns a FuncItem that corresponds to the output of the decorator. @@ -1079,6 +1125,8 @@ def verify_decorator( if stub.func.is_property: for message in _verify_readonly_property(stub, runtime): yield Error(object_path, message, stub, runtime) + for message in _verify_abstract_status(stub.func, runtime): + yield Error(object_path, message, stub, runtime) return func = _resolve_funcitem_from_decorator(stub) @@ -1270,16 +1318,13 @@ def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool: isinstance(left, mypy.types.LiteralType) and isinstance(left.value, int) and left.value in (0, 1) - and isinstance(right, mypy.types.Instance) - and right.type.fullname == "builtins.bool" + and mypy.types.is_named_instance(right, "builtins.bool") ): # Pretend Literal[0, 1] is a subtype of bool to avoid unhelpful errors. return True - if ( - isinstance(right, mypy.types.TypedDictType) - and isinstance(left, mypy.types.Instance) - and left.type.fullname == "builtins.dict" + if isinstance(right, mypy.types.TypedDictType) and mypy.types.is_named_instance( + left, "builtins.dict" ): # Special case checks against TypedDicts return True @@ -1555,6 +1600,8 @@ def test_stubs(args: _Arguments, use_builtins_fixtures: bool = False) -> int: options = Options() options.incremental = False options.custom_typeshed_dir = args.custom_typeshed_dir + if options.custom_typeshed_dir: + options.abs_custom_typeshed_dir = os.path.abspath(args.custom_typeshed_dir) options.config_file = args.mypy_config_file options.use_builtins_fixtures = use_builtins_fixtures @@ -1687,7 +1734,7 @@ def parse_options(args: list[str]) -> _Arguments: parser.add_argument( "--mypy-config-file", metavar="FILE", - help=("Use specified mypy config file to determine mypy plugins " "and mypy path"), + help=("Use specified mypy config file to determine mypy plugins and mypy path"), ) parser.add_argument( "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR" diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 9e84e25695dd..38fae16e7011 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -38,6 +38,7 @@ Instance, LiteralType, NoneType, + NormalizedCallableType, Overloaded, Parameters, ParamSpecType, @@ -60,14 +61,16 @@ is_named_instance, ) from mypy.typestate import SubtypeKind, TypeState -from mypy.typevartuples import extract_unpack, split_with_instance +from mypy.typevars import fill_typevars_with_any +from mypy.typevartuples import extract_unpack, fully_split_with_mapped_and_template # Flags for detected protocol members IS_SETTABLE: Final = 1 IS_CLASSVAR: Final = 2 IS_CLASS_OR_STATIC: Final = 3 +IS_VAR: Final = 4 -TypeParameterChecker: _TypeAlias = Callable[[Type, Type, int, bool], bool] +TypeParameterChecker: _TypeAlias = Callable[[Type, Type, int, bool, "SubtypeContext"], bool] class SubtypeContext: @@ -80,6 +83,7 @@ def __init__( ignore_declared_variance: bool = False, # Supported for both proper and non-proper ignore_promotions: bool = False, + ignore_uninhabited: bool = False, # Proper subtype flags erase_instances: bool = False, keep_erased_types: bool = False, @@ -89,6 +93,7 @@ def __init__( self.ignore_pos_arg_names = ignore_pos_arg_names self.ignore_declared_variance = ignore_declared_variance self.ignore_promotions = ignore_promotions + self.ignore_uninhabited = ignore_uninhabited self.erase_instances = erase_instances self.keep_erased_types = keep_erased_types self.options = options @@ -115,6 +120,7 @@ def is_subtype( ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, ignore_promotions: bool = False, + ignore_uninhabited: bool = False, options: Options | None = None, ) -> bool: """Is 'left' subtype of 'right'? @@ -134,6 +140,7 @@ def is_subtype( ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, ignore_promotions=ignore_promotions, + ignore_uninhabited=ignore_uninhabited, options=options, ) else: @@ -143,6 +150,7 @@ def is_subtype( ignore_pos_arg_names, ignore_declared_variance, ignore_promotions, + ignore_uninhabited, options, } ), "Don't pass both context and individual flags" @@ -177,6 +185,7 @@ def is_proper_subtype( *, subtype_context: SubtypeContext | None = None, ignore_promotions: bool = False, + ignore_uninhabited: bool = False, erase_instances: bool = False, keep_erased_types: bool = False, ) -> bool: @@ -192,12 +201,19 @@ def is_proper_subtype( if subtype_context is None: subtype_context = SubtypeContext( ignore_promotions=ignore_promotions, + ignore_uninhabited=ignore_uninhabited, erase_instances=erase_instances, keep_erased_types=keep_erased_types, ) else: assert not any( - {ignore_promotions, erase_instances, keep_erased_types} + { + ignore_promotions, + ignore_uninhabited, + erase_instances, + keep_erased_types, + ignore_uninhabited, + } ), "Don't pass both context and individual flags" if TypeState.is_assumed_proper_subtype(left, right): return True @@ -215,6 +231,7 @@ def is_equivalent( ignore_type_params: bool = False, ignore_pos_arg_names: bool = False, options: Options | None = None, + subtype_context: SubtypeContext | None = None, ) -> bool: return is_subtype( a, @@ -222,16 +239,20 @@ def is_equivalent( ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, options=options, + subtype_context=subtype_context, ) and is_subtype( b, a, ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, options=options, + subtype_context=subtype_context, ) -def is_same_type(a: Type, b: Type, ignore_promotions: bool = True) -> bool: +def is_same_type( + a: Type, b: Type, ignore_promotions: bool = True, subtype_context: SubtypeContext | None = None +) -> bool: """Are these types proper subtypes of each other? This means types may have different representation (e.g. an alias, or @@ -241,8 +262,10 @@ def is_same_type(a: Type, b: Type, ignore_promotions: bool = True) -> bool: # considered not the same type (which is the case at runtime). # Also Union[bool, int] (if it wasn't simplified before) will be different # from plain int, etc. - return is_proper_subtype(a, b, ignore_promotions=ignore_promotions) and is_proper_subtype( - b, a, ignore_promotions=ignore_promotions + return is_proper_subtype( + a, b, ignore_promotions=ignore_promotions, subtype_context=subtype_context + ) and is_proper_subtype( + b, a, ignore_promotions=ignore_promotions, subtype_context=subtype_context ) @@ -306,11 +329,15 @@ def check_item(left: Type, right: Type, subtype_context: SubtypeContext) -> bool return left.accept(SubtypeVisitor(orig_right, subtype_context, proper_subtype)) -# TODO: should we pass on the original flags here and in couple other places? -# This seems logical but was never done in the past for some reasons. -def check_type_parameter(lefta: Type, righta: Type, variance: int, proper_subtype: bool) -> bool: +def check_type_parameter( + lefta: Type, righta: Type, variance: int, proper_subtype: bool, subtype_context: SubtypeContext +) -> bool: def check(left: Type, right: Type) -> bool: - return is_proper_subtype(left, right) if proper_subtype else is_subtype(left, right) + return ( + is_proper_subtype(left, right, subtype_context=subtype_context) + if proper_subtype + else is_subtype(left, right, subtype_context=subtype_context) + ) if variance == COVARIANT: return check(lefta, righta) @@ -318,11 +345,18 @@ def check(left: Type, right: Type) -> bool: return check(righta, lefta) else: if proper_subtype: - return is_same_type(lefta, righta) - return is_equivalent(lefta, righta) + # We pass ignore_promotions=False because it is a default for subtype checks. + # The actual value will be taken from the subtype_context, and it is whatever + # the original caller passed. + return is_same_type( + lefta, righta, ignore_promotions=False, subtype_context=subtype_context + ) + return is_equivalent(lefta, righta, subtype_context=subtype_context) -def ignore_type_parameter(lefta: Type, righta: Type, variance: int, proper_subtype: bool) -> bool: +def ignore_type_parameter( + lefta: Type, righta: Type, variance: int, proper_subtype: bool, subtype_context: SubtypeContext +) -> bool: return True @@ -385,7 +419,11 @@ def visit_none_type(self, left: NoneType) -> bool: return True def visit_uninhabited_type(self, left: UninhabitedType) -> bool: - return True + # We ignore this for unsafe overload checks, so that and empty list and + # a list of int will be considered non-overlapping. + if isinstance(self.right, UninhabitedType): + return True + return not self.subtype_context.ignore_uninhabited def visit_erased_type(self, left: ErasedType) -> bool: # This may be encountered during type inference. The result probably doesn't @@ -447,8 +485,22 @@ def visit_instance(self, left: Instance) -> bool: t = erased nominal = True if right.type.has_type_var_tuple_type: - left_prefix, left_middle, left_suffix = split_with_instance(left) - right_prefix, right_middle, right_suffix = split_with_instance(right) + split_result = fully_split_with_mapped_and_template(left, right) + if split_result is None: + return False + + ( + left_prefix, + left_mprefix, + left_middle, + left_msuffix, + left_suffix, + right_prefix, + right_mprefix, + right_middle, + right_msuffix, + right_suffix, + ) = split_result left_unpacked = extract_unpack(left_middle) right_unpacked = extract_unpack(right_middle) @@ -457,6 +509,15 @@ def visit_instance(self, left: Instance) -> bool: def check_mixed( unpacked_type: ProperType, compare_to: tuple[Type, ...] ) -> bool: + if ( + isinstance(unpacked_type, Instance) + and unpacked_type.type.fullname == "builtins.tuple" + ): + if not all( + is_equivalent(l, unpacked_type.args[0]) for l in compare_to + ): + return False + return True if isinstance(unpacked_type, TypeVarTupleType): return False if isinstance(unpacked_type, AnyType): @@ -483,13 +544,6 @@ def check_mixed( if not check_mixed(left_unpacked, right_middle): return False elif left_unpacked is None and right_unpacked is not None: - if ( - isinstance(right_unpacked, Instance) - and right_unpacked.type.fullname == "builtins.tuple" - ): - return all( - is_equivalent(l, right_unpacked.args[0]) for l in left_middle - ) if not check_mixed(right_unpacked, left_middle): return False @@ -502,16 +556,24 @@ def check_mixed( if not is_equivalent(left_t, right_t): return False + assert len(left_mprefix) == len(right_mprefix) + assert len(left_msuffix) == len(right_msuffix) + + for left_item, right_item in zip( + left_mprefix + left_msuffix, right_mprefix + right_msuffix + ): + if not is_equivalent(left_item, right_item): + return False + left_items = t.args[: right.type.type_var_tuple_prefix] right_items = right.args[: right.type.type_var_tuple_prefix] if right.type.type_var_tuple_suffix: left_items += t.args[-right.type.type_var_tuple_suffix :] right_items += right.args[-right.type.type_var_tuple_suffix :] - unpack_index = right.type.type_var_tuple_prefix assert unpack_index is not None type_params = zip( - left_prefix + right_suffix, + left_prefix + left_suffix, right_prefix + right_suffix, right.type.defn.type_vars[:unpack_index] + right.type.defn.type_vars[unpack_index + 1 :], @@ -521,12 +583,12 @@ def check_mixed( for lefta, righta, tvar in type_params: if isinstance(tvar, TypeVarType): if not self.check_type_parameter( - lefta, righta, tvar.variance, self.proper_subtype + lefta, righta, tvar.variance, self.proper_subtype, self.subtype_context ): nominal = False else: if not self.check_type_parameter( - lefta, righta, COVARIANT, self.proper_subtype + lefta, righta, COVARIANT, self.proper_subtype, self.subtype_context ): nominal = False if nominal: @@ -591,8 +653,10 @@ def visit_unpack_type(self, left: UnpackType) -> bool: return False def visit_parameters(self, left: Parameters) -> bool: - right = self.right - if isinstance(right, Parameters) or isinstance(right, CallableType): + if isinstance(self.right, Parameters) or isinstance(self.right, CallableType): + right = self.right + if isinstance(right, CallableType): + right = right.with_unpacked_kwargs() return are_parameters_compatible( left, right, @@ -629,6 +693,14 @@ def visit_callable_type(self, left: CallableType) -> bool: assert call is not None if self._is_subtype(left, call): return True + if right.type.is_protocol and left.is_type_obj(): + ret_type = get_proper_type(left.ret_type) + if isinstance(ret_type, TupleType): + ret_type = mypy.typeops.tuple_fallback(ret_type) + if isinstance(ret_type, Instance) and is_protocol_implementation( + ret_type, right, proper_subtype=self.proper_subtype, class_obj=True + ): + return True return self._is_subtype(left.fallback, right) elif isinstance(right, TypeType): # This is unsound, we don't check the __init__ signature. @@ -636,7 +708,7 @@ def visit_callable_type(self, left: CallableType) -> bool: elif isinstance(right, Parameters): # this doesn't check return types.... but is needed for is_equivalent return are_parameters_compatible( - left, + left.with_unpacked_kwargs(), right, is_compat=self._is_subtype, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, @@ -694,6 +766,7 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: if not left.names_are_wider_than(right): return False for name, l, r in left.zip(right): + # TODO: should we pass on the full subtype_context here and below? if self.proper_subtype: check = is_same_type(l, r) else: @@ -844,6 +917,10 @@ def visit_type_type(self, left: TypeType) -> bool: if isinstance(right, TypeType): return self._is_subtype(left.item, right.item) if isinstance(right, CallableType): + if self.proper_subtype and not right.is_type_obj(): + # We can't accept `Type[X]` as a *proper* subtype of Callable[P, X] + # since this will break transitivity of subtyping. + return False # This is unsound, we don't check the __init__ signature. return self._is_subtype(left.item, right.ret_type) if isinstance(right, Instance): @@ -857,6 +934,10 @@ def visit_type_type(self, left: TypeType) -> bool: if isinstance(item, TypeVarType): item = get_proper_type(item.upper_bound) if isinstance(item, Instance): + if right.type.is_protocol and is_protocol_implementation( + item, right, proper_subtype=self.proper_subtype, class_obj=True + ): + return True metaclass = item.type.metaclass_type return metaclass is not None and self._is_subtype(metaclass, right) return False @@ -876,7 +957,7 @@ def pop_on_exit(stack: list[tuple[T, T]], left: T, right: T) -> Iterator[None]: def is_protocol_implementation( - left: Instance, right: Instance, proper_subtype: bool = False + left: Instance, right: Instance, proper_subtype: bool = False, class_obj: bool = False ) -> bool: """Check whether 'left' implements the protocol 'right'. @@ -919,7 +1000,19 @@ def f(self) -> A: ... # We always bind self to the subtype. (Similarly to nominal types). supertype = get_proper_type(find_member(member, right, left)) assert supertype is not None - subtype = get_proper_type(find_member(member, left, left)) + if member == "__call__" and class_obj: + # Special case: class objects always have __call__ that is just the constructor. + # TODO: move this helper function to typeops.py? + import mypy.checkmember + + def named_type(fullname: str) -> Instance: + return Instance(left.type.mro[-1], []) + + subtype: ProperType | None = mypy.checkmember.type_object_type( + left.type, named_type + ) + else: + subtype = get_proper_type(find_member(member, left, left, class_obj=class_obj)) # Useful for debugging: # print(member, 'of', left, 'has type', subtype) # print(member, 'of', right, 'has type', supertype) @@ -946,14 +1039,22 @@ def f(self) -> A: ... if isinstance(subtype, NoneType) and isinstance(supertype, CallableType): # We want __hash__ = None idiom to work even without --strict-optional return False - subflags = get_member_flags(member, left.type) - superflags = get_member_flags(member, right.type) + subflags = get_member_flags(member, left, class_obj=class_obj) + superflags = get_member_flags(member, right) if IS_SETTABLE in superflags: # Check opposite direction for settable attributes. if not is_subtype(supertype, subtype): return False - if (IS_CLASSVAR in subflags) != (IS_CLASSVAR in superflags): - return False + if not class_obj: + if (IS_CLASSVAR in subflags) != (IS_CLASSVAR in superflags): + return False + else: + if IS_VAR in superflags and IS_CLASSVAR not in subflags: + # Only class variables are allowed for class object access. + return False + if IS_CLASSVAR in superflags: + # This can be never matched by a class object. + return False if IS_SETTABLE in superflags and IS_SETTABLE not in subflags: return False # This rule is copied from nominal check in checker.py @@ -974,7 +1075,7 @@ def f(self) -> A: ... def find_member( - name: str, itype: Instance, subtype: Type, is_operator: bool = False + name: str, itype: Instance, subtype: Type, is_operator: bool = False, class_obj: bool = False ) -> Type | None: """Find the type of member by 'name' in 'itype's TypeInfo. @@ -987,23 +1088,25 @@ def find_member( method = info.get_method(name) if method: if isinstance(method, Decorator): - return find_node_type(method.var, itype, subtype) + return find_node_type(method.var, itype, subtype, class_obj=class_obj) if method.is_property: assert isinstance(method, OverloadedFuncDef) dec = method.items[0] assert isinstance(dec, Decorator) - return find_node_type(dec.var, itype, subtype) - return find_node_type(method, itype, subtype) + return find_node_type(dec.var, itype, subtype, class_obj=class_obj) + return find_node_type(method, itype, subtype, class_obj=class_obj) else: # don't have such method, maybe variable or decorator? node = info.get(name) v = node.node if node else None if isinstance(v, Var): - return find_node_type(v, itype, subtype) + return find_node_type(v, itype, subtype, class_obj=class_obj) if ( not v and name not in ["__getattr__", "__setattr__", "__getattribute__"] and not is_operator + and not class_obj + and itype.extra_attrs is None # skip ModuleType.__getattr__ ): for method_name in ("__getattribute__", "__getattr__"): # Normally, mypy assumes that instances that define __getattr__ have all @@ -1021,10 +1124,16 @@ def find_member( return getattr_type if itype.type.fallback_to_any: return AnyType(TypeOfAny.special_form) + if isinstance(v, TypeInfo): + # PEP 544 doesn't specify anything about such use cases. So we just try + # to do something meaningful (at least we should not crash). + return TypeType(fill_typevars_with_any(v)) + if itype.extra_attrs and name in itype.extra_attrs.attrs: + return itype.extra_attrs.attrs[name] return None -def get_member_flags(name: str, info: TypeInfo) -> set[int]: +def get_member_flags(name: str, itype: Instance, class_obj: bool = False) -> set[int]: """Detect whether a member 'name' is settable, whether it is an instance or class variable, and whether it is class or static method. @@ -1035,35 +1144,53 @@ def get_member_flags(name: str, info: TypeInfo) -> set[int]: * IS_CLASS_OR_STATIC: set for methods decorated with @classmethod or with @staticmethod. """ + info = itype.type method = info.get_method(name) setattr_meth = info.get_method("__setattr__") if method: if isinstance(method, Decorator): if method.var.is_staticmethod or method.var.is_classmethod: return {IS_CLASS_OR_STATIC} + elif method.var.is_property: + return {IS_VAR} elif method.is_property: # this could be settable property assert isinstance(method, OverloadedFuncDef) dec = method.items[0] assert isinstance(dec, Decorator) if dec.var.is_settable_property or setattr_meth: - return {IS_SETTABLE} - return set() + return {IS_VAR, IS_SETTABLE} + else: + return {IS_VAR} + return set() # Just a regular method node = info.get(name) if not node: if setattr_meth: return {IS_SETTABLE} + if itype.extra_attrs and name in itype.extra_attrs.attrs: + flags = set() + if name not in itype.extra_attrs.immutable: + flags.add(IS_SETTABLE) + return flags return set() v = node.node # just a variable - if isinstance(v, Var) and not v.is_property: - flags = {IS_SETTABLE} + if isinstance(v, Var): + if v.is_property: + return {IS_VAR} + flags = {IS_VAR} + if not v.is_final: + flags.add(IS_SETTABLE) if v.is_classvar: flags.add(IS_CLASSVAR) + if class_obj and v.is_inferred: + flags.add(IS_CLASSVAR) return flags return set() -def find_node_type(node: Var | FuncBase, itype: Instance, subtype: Type) -> Type: +def find_node_type( + node: Var | FuncBase, itype: Instance, subtype: Type, class_obj: bool = False +) -> Type: """Find type of a variable or method 'node' (maybe also a decorated method). Apply type arguments from 'itype', and bind 'self' to 'subtype'. """ @@ -1085,10 +1212,16 @@ def find_node_type(node: Var | FuncBase, itype: Instance, subtype: Type) -> Type and not node.is_staticmethod ): assert isinstance(p_typ, FunctionLike) - signature = bind_self( - p_typ, subtype, is_classmethod=isinstance(node, Var) and node.is_classmethod - ) - if node.is_property: + if class_obj and not ( + node.is_class if isinstance(node, FuncBase) else node.is_classmethod + ): + # Don't bind instance methods on class objects. + signature = p_typ + else: + signature = bind_self( + p_typ, subtype, is_classmethod=isinstance(node, Var) and node.is_classmethod + ) + if node.is_property and not class_obj: assert isinstance(signature, CallableType) typ = signature.ret_type else: @@ -1213,6 +1346,10 @@ def g(x: int) -> int: ... If the 'some_check' function is also symmetric, the two calls would be equivalent whether or not we check the args covariantly. """ + # Normalize both types before comparing them. + left = left.with_unpacked_kwargs() + right = right.with_unpacked_kwargs() + if is_compat_return is None: is_compat_return = is_compat @@ -1277,8 +1414,8 @@ def g(x: int) -> int: ... def are_parameters_compatible( - left: Parameters | CallableType, - right: Parameters | CallableType, + left: Parameters | NormalizedCallableType, + right: Parameters | NormalizedCallableType, *, is_compat: Callable[[Type, Type], bool], ignore_pos_arg_names: bool = False, @@ -1499,11 +1636,11 @@ def new_is_compat(left: Type, right: Type) -> bool: def unify_generic_callable( - type: CallableType, - target: CallableType, + type: NormalizedCallableType, + target: NormalizedCallableType, ignore_return: bool, return_constraint_direction: int | None = None, -) -> CallableType | None: +) -> NormalizedCallableType | None: """Try to unify a generic callable type with another callable type. Return unified CallableType if successful; otherwise, return None. @@ -1540,7 +1677,7 @@ def report(*args: Any) -> None: ) if had_errors: return None - return applied + return cast(NormalizedCallableType, applied) def try_restrict_literal_union(t: UnionType, s: Type) -> list[Type] | None: @@ -1564,35 +1701,32 @@ def try_restrict_literal_union(t: UnionType, s: Type) -> list[Type] | None: return new_items -def restrict_subtype_away(t: Type, s: Type, *, ignore_promotions: bool = False) -> Type: +def restrict_subtype_away(t: Type, s: Type) -> Type: """Return t minus s for runtime type assertions. If we can't determine a precise result, return a supertype of the ideal result (just t is a valid result). This is used for type inference of runtime type checks such as - isinstance(). Currently this just removes elements of a union type. + isinstance(). Currently, this just removes elements of a union type. """ p_t = get_proper_type(t) if isinstance(p_t, UnionType): new_items = try_restrict_literal_union(p_t, s) if new_items is None: new_items = [ - restrict_subtype_away(item, s, ignore_promotions=ignore_promotions) + restrict_subtype_away(item, s) for item in p_t.relevant_items() - if ( - isinstance(get_proper_type(item), AnyType) - or not covers_at_runtime(item, s, ignore_promotions) - ) + if (isinstance(get_proper_type(item), AnyType) or not covers_at_runtime(item, s)) ] return UnionType.make_union(new_items) - elif covers_at_runtime(t, s, ignore_promotions): + elif covers_at_runtime(t, s): return UninhabitedType() else: return t -def covers_at_runtime(item: Type, supertype: Type, ignore_promotions: bool) -> bool: +def covers_at_runtime(item: Type, supertype: Type) -> bool: """Will isinstance(item, supertype) always return True at runtime?""" item = get_proper_type(item) supertype = get_proper_type(supertype) @@ -1600,12 +1734,12 @@ def covers_at_runtime(item: Type, supertype: Type, ignore_promotions: bool) -> b # Since runtime type checks will ignore type arguments, erase the types. supertype = erase_type(supertype) if is_proper_subtype( - erase_type(item), supertype, ignore_promotions=ignore_promotions, erase_instances=True + erase_type(item), supertype, ignore_promotions=True, erase_instances=True ): return True if isinstance(supertype, Instance) and supertype.type.is_protocol: # TODO: Implement more robust support for runtime isinstance() checks, see issue #3827. - if is_proper_subtype(item, supertype, ignore_promotions=ignore_promotions): + if is_proper_subtype(item, supertype, ignore_promotions=True): return True if isinstance(item, TypedDictType) and isinstance(supertype, Instance): # Special case useful for selecting TypedDicts from unions using isinstance(x, dict). diff --git a/mypy/test/data.py b/mypy/test/data.py index e08b95fedbde..f4cb39818b4e 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -189,7 +189,7 @@ def parse_test_case(case: DataDrivenTestCase) -> None: ): raise ValueError( ( - "Stale modules after pass {} must be a subset of rechecked " "modules ({}:{})" + "Stale modules after pass {} must be a subset of rechecked modules ({}:{})" ).format(passnum, case.file, first_item.line) ) @@ -680,7 +680,9 @@ class DataFileCollector(pytest.Collector): def from_parent( # type: ignore[override] cls, parent: DataSuiteCollector, *, name: str ) -> DataFileCollector: - return super().from_parent(parent, name=name) + collector = super().from_parent(parent, name=name) + assert isinstance(collector, DataFileCollector) + return collector def collect(self) -> Iterator[DataDrivenTestCase]: yield from split_test_cases( diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index cd3ae4b71071..145027404ff7 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -282,8 +282,14 @@ def num_skipped_suffix_lines(a1: list[str], a2: list[str]) -> int: def testfile_pyversion(path: str) -> tuple[int, int]: - if path.endswith("python310.test"): + if path.endswith("python311.test"): + return 3, 11 + elif path.endswith("python310.test"): return 3, 10 + elif path.endswith("python39.test"): + return 3, 9 + elif path.endswith("python38.test"): + return 3, 8 else: return defaults.PYTHON3_VERSION @@ -369,12 +375,15 @@ def parse_options( if targets: # TODO: support specifying targets via the flags pragma raise RuntimeError("Specifying targets via the flags pragma is not supported.") + if "--show-error-codes" not in flag_list: + options.hide_error_codes = True else: flag_list = [] options = Options() # TODO: Enable strict optional in test cases by default (requires *many* test case changes) options.strict_optional = False options.error_summary = False + options.hide_error_codes = True # Allow custom python version to override testfile_pyversion. if all(flag.split("=")[0] not in ["--python-version", "-2", "--py2"] for flag in flag_list): diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 97a2ed664454..21ba0903a824 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -356,7 +356,8 @@ def test_find_sources_exclude(self) -> None: "/kg", "/g.py", "/bc", - "/xxx/pkg/a2/b/f.py" "xxx/pkg/a2/b/f.py", + "/xxx/pkg/a2/b/f.py", + "xxx/pkg/a2/b/f.py", ] big_exclude2 = ["|".join(big_exclude1)] for big_exclude in [big_exclude1, big_exclude2]: diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index cae427de2f96..4fe2ee6393c0 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -10,6 +10,7 @@ from mypy.build import Graph from mypy.errors import CompileError from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths +from mypy.options import TYPE_VAR_TUPLE, UNPACK from mypy.semanal_main import core_modules from mypy.test.config import test_data_prefix, test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, module_from_path @@ -26,7 +27,7 @@ ) try: - import lxml # type: ignore + import lxml # type: ignore[import] except ImportError: lxml = None @@ -43,6 +44,8 @@ typecheck_files.remove("check-python39.test") if sys.version_info < (3, 10): typecheck_files.remove("check-python310.test") +if sys.version_info < (3, 11): + typecheck_files.remove("check-python311.test") # Special tests for platforms with case-insensitive filesystems. if sys.platform not in ("darwin", "win32"): @@ -110,7 +113,8 @@ def run_case_once( # Parse options after moving files (in case mypy.ini is being moved). options = parse_options(original_program_text, testcase, incremental_step) options.use_builtins_fixtures = True - options.enable_incomplete_features = True + if not testcase.name.endswith("_no_incomplete"): + options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] options.show_traceback = True # Enable some options automatically based on test file name. @@ -119,7 +123,9 @@ def run_case_once( if "columns" in testcase.file: options.show_column_numbers = True if "errorcodes" in testcase.file: - options.show_error_codes = True + options.hide_error_codes = False + if "abstract" not in testcase.file: + options.allow_empty_bodies = not testcase.name.endswith("_no_empty") if incremental_step and options.incremental: # Don't overwrite # flags: --no-incremental in incremental test cases diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 14c985e1d9a9..268b6bab1ec2 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -20,7 +20,7 @@ ) try: - import lxml # type: ignore + import lxml # type: ignore[import] except ImportError: lxml = None @@ -57,6 +57,10 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None: args.append("--show-traceback") if "--error-summary" not in args: args.append("--no-error-summary") + if "--show-error-codes" not in args: + args.append("--hide-error-codes") + if "--disallow-empty-bodies" not in args: + args.append("--allow-empty-bodies") # Type check the program. fixed = [python3_path, "-m", "mypy"] env = os.environ.copy() diff --git a/mypy/test/testconstraints.py b/mypy/test/testconstraints.py index 4aa9fdb83a29..6b8f596dd605 100644 --- a/mypy/test/testconstraints.py +++ b/mypy/test/testconstraints.py @@ -5,7 +5,7 @@ from mypy.constraints import SUBTYPE_OF, SUPERTYPE_OF, Constraint, infer_constraints from mypy.test.helpers import Suite from mypy.test.typefixture import TypeFixture -from mypy.types import Instance, TypeList, UnpackType +from mypy.types import Instance, TupleType, TypeList, UnpackType class ConstraintsSuite(Suite): @@ -19,7 +19,7 @@ def test_basic_type_variable(self) -> None: fx = self.fx for direction in [SUBTYPE_OF, SUPERTYPE_OF]: assert infer_constraints(fx.gt, fx.ga, direction) == [ - Constraint(type_var=fx.t.id, op=direction, target=fx.a) + Constraint(type_var=fx.t, op=direction, target=fx.a) ] @pytest.mark.xfail @@ -27,13 +27,13 @@ def test_basic_type_var_tuple_subtype(self) -> None: fx = self.fx assert infer_constraints( Instance(fx.gvi, [UnpackType(fx.ts)]), Instance(fx.gvi, [fx.a, fx.b]), SUBTYPE_OF - ) == [Constraint(type_var=fx.ts.id, op=SUBTYPE_OF, target=TypeList([fx.a, fx.b]))] + ) == [Constraint(type_var=fx.ts, op=SUBTYPE_OF, target=TypeList([fx.a, fx.b]))] def test_basic_type_var_tuple(self) -> None: fx = self.fx assert infer_constraints( Instance(fx.gvi, [UnpackType(fx.ts)]), Instance(fx.gvi, [fx.a, fx.b]), SUPERTYPE_OF - ) == [Constraint(type_var=fx.ts.id, op=SUPERTYPE_OF, target=TypeList([fx.a, fx.b]))] + ) == [Constraint(type_var=fx.ts, op=SUPERTYPE_OF, target=TypeList([fx.a, fx.b]))] def test_type_var_tuple_with_prefix_and_suffix(self) -> None: fx = self.fx @@ -44,7 +44,102 @@ def test_type_var_tuple_with_prefix_and_suffix(self) -> None: SUPERTYPE_OF, ) ) == { - Constraint(type_var=fx.t.id, op=SUPERTYPE_OF, target=fx.a), - Constraint(type_var=fx.ts.id, op=SUPERTYPE_OF, target=TypeList([fx.b, fx.c])), - Constraint(type_var=fx.s.id, op=SUPERTYPE_OF, target=fx.d), + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.ts, op=SUPERTYPE_OF, target=TypeList([fx.b, fx.c])), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.d), + } + + def test_unpack_homogenous_tuple(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance(fx.gvi, [UnpackType(Instance(fx.std_tuplei, [fx.t]))]), + Instance(fx.gvi, [fx.a, fx.b]), + SUPERTYPE_OF, + ) + ) == { + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.b), + } + + def test_unpack_homogenous_tuple_with_prefix_and_suffix(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance(fx.gv2i, [fx.t, UnpackType(Instance(fx.std_tuplei, [fx.s])), fx.u]), + Instance(fx.gv2i, [fx.a, fx.b, fx.c, fx.d]), + SUPERTYPE_OF, + ) + ) == { + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.b), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.c), + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), + } + + def test_unpack_tuple(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance( + fx.gvi, + [ + UnpackType( + TupleType([fx.t, fx.s], fallback=Instance(fx.std_tuplei, [fx.o])) + ) + ], + ), + Instance(fx.gvi, [fx.a, fx.b]), + SUPERTYPE_OF, + ) + ) == { + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.b), + } + + def test_unpack_with_prefix_and_suffix(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance( + fx.gv2i, + [ + fx.u, + UnpackType( + TupleType([fx.t, fx.s], fallback=Instance(fx.std_tuplei, [fx.o])) + ), + fx.u, + ], + ), + Instance(fx.gv2i, [fx.a, fx.b, fx.c, fx.d]), + SUPERTYPE_OF, + ) + ) == { + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.b), + Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.c), + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), + } + + def test_unpack_tuple_length_non_match(self) -> None: + fx = self.fx + assert set( + infer_constraints( + Instance( + fx.gv2i, + [ + fx.u, + UnpackType( + TupleType([fx.t, fx.s], fallback=Instance(fx.std_tuplei, [fx.o])) + ), + fx.u, + ], + ), + Instance(fx.gv2i, [fx.a, fx.b, fx.d]), + SUPERTYPE_OF, + ) + # We still get constraints on the prefix/suffix in this case. + ) == { + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), } diff --git a/mypy/test/testdaemon.py b/mypy/test/testdaemon.py index 04a9c387b68a..e3cdf44d89f2 100644 --- a/mypy/test/testdaemon.py +++ b/mypy/test/testdaemon.py @@ -81,6 +81,8 @@ def parse_script(input: list[str]) -> list[list[str]]: def run_cmd(input: str) -> tuple[int, str]: + if input[1:].startswith("mypy run --") and "--show-error-codes" not in input: + input += " --hide-error-codes" if input.startswith("dmypy "): input = sys.executable + " -m mypy." + input if input.startswith("mypy "): diff --git a/mypy/test/testdeps.py b/mypy/test/testdeps.py index 7cbe619bad09..ae1c613f7563 100644 --- a/mypy/test/testdeps.py +++ b/mypy/test/testdeps.py @@ -4,7 +4,6 @@ import os from collections import defaultdict -from typing import DefaultDict from mypy import build from mypy.errors import CompileError @@ -34,13 +33,14 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: options.cache_dir = os.devnull options.export_types = True options.preserve_asts = True + options.allow_empty_bodies = True messages, files, type_map = self.build(src, options) a = messages if files is None or type_map is None: if not a: a = ["Unknown compile error (likely syntax error in test case or fixture)"] else: - deps: DefaultDict[str, set[str]] = defaultdict(set) + deps: defaultdict[str, set[str]] = defaultdict(set) for module in files: if ( module in dumped_modules diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py index 4ef82720fdcb..5e2e0bc2ca5a 100644 --- a/mypy/test/testdiff.py +++ b/mypy/test/testdiff.py @@ -54,6 +54,7 @@ def build(self, source: str, options: Options) -> tuple[list[str], dict[str, Myp options.show_traceback = True options.cache_dir = os.devnull options.python_version = PYTHON3_VERSION + options.allow_empty_bodies = True try: result = build.build( sources=[BuildSource("main", None, source)], diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index bae26b148a79..4b98f10fc9ca 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -25,6 +25,7 @@ def test_error_stream(testcase: DataDrivenTestCase) -> None: """ options = Options() options.show_traceback = True + options.hide_error_codes = True logged_messages: list[str] = [] diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index 1cc8ba6198d1..b19c49bf60bc 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -17,6 +17,7 @@ import os import re import sys +import unittest from typing import Any, cast import pytest @@ -28,8 +29,9 @@ from mypy.errors import CompileError from mypy.find_sources import create_source_list from mypy.modulefinder import BuildSource -from mypy.options import Options +from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options from mypy.server.mergecheck import check_consistency +from mypy.server.update import sort_messages_preserving_file_order from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite, DeleteFile, UpdateFile from mypy.test.helpers import ( @@ -151,6 +153,9 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo options.use_fine_grained_cache = self.use_cache and not build_cache options.cache_fine_grained = self.use_cache options.local_partial_types = True + options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] + # Treat empty bodies safely for these test cases. + options.allow_empty_bodies = not testcase.name.endswith("_no_empty") if re.search("flags:.*--follow-imports", source) is None: # Override the default for follow_imports options.follow_imports = "error" @@ -366,3 +371,70 @@ def get_inspect(self, program_text: str, incremental_step: int) -> list[tuple[st def normalize_messages(messages: list[str]) -> list[str]: return [re.sub("^tmp" + re.escape(os.sep), "", message) for message in messages] + + +class TestMessageSorting(unittest.TestCase): + def test_simple_sorting(self) -> None: + msgs = ['x.py:1: error: "int" not callable', 'foo/y.py:123: note: "X" not defined'] + old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable'] + assert sort_messages_preserving_file_order(msgs, old_msgs) == list(reversed(msgs)) + assert sort_messages_preserving_file_order(list(reversed(msgs)), old_msgs) == list( + reversed(msgs) + ) + + def test_long_form_sorting(self) -> None: + # Multi-line errors should be sorted together and not split. + msg1 = [ + 'x.py:1: error: "int" not callable', + "and message continues (x: y)", + " 1()", + " ^~~", + ] + msg2 = [ + 'foo/y.py: In function "f":', + 'foo/y.py:123: note: "X" not defined', + "and again message continues", + ] + old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable'] + assert sort_messages_preserving_file_order(msg1 + msg2, old_msgs) == msg2 + msg1 + assert sort_messages_preserving_file_order(msg2 + msg1, old_msgs) == msg2 + msg1 + + def test_mypy_error_prefix(self) -> None: + # Some errors don't have a file and start with "mypy: ". These + # shouldn't be sorted together with file-specific errors. + msg1 = 'x.py:1: error: "int" not callable' + msg2 = 'foo/y:123: note: "X" not defined' + msg3 = "mypy: Error not associated with a file" + old_msgs = [ + "mypy: Something wrong", + 'foo/y:12: note: "Y" not defined', + 'x.py:8: error: "str" not callable', + ] + assert sort_messages_preserving_file_order([msg1, msg2, msg3], old_msgs) == [ + msg2, + msg1, + msg3, + ] + assert sort_messages_preserving_file_order([msg3, msg2, msg1], old_msgs) == [ + msg2, + msg1, + msg3, + ] + + def test_new_file_at_the_end(self) -> None: + msg1 = 'x.py:1: error: "int" not callable' + msg2 = 'foo/y.py:123: note: "X" not defined' + new1 = "ab.py:3: error: Problem: error" + new2 = "aaa:3: error: Bad" + old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable'] + assert sort_messages_preserving_file_order([msg1, msg2, new1], old_msgs) == [ + msg2, + msg1, + new1, + ] + assert sort_messages_preserving_file_order([new1, msg1, msg2, new2], old_msgs) == [ + msg2, + msg1, + new1, + new2, + ] diff --git a/mypy/test/testinfer.py b/mypy/test/testinfer.py index cf6d648dba5a..08926c179623 100644 --- a/mypy/test/testinfer.py +++ b/mypy/test/testinfer.py @@ -2,8 +2,6 @@ from __future__ import annotations -from typing import Tuple - from mypy.argmap import map_actuals_to_formals from mypy.checker import DisjointDict, group_comparison_operands from mypy.literals import Key @@ -46,15 +44,18 @@ def test_too_many_caller_args(self) -> None: def test_tuple_star(self) -> None: any_type = AnyType(TypeOfAny.special_form) - self.assert_vararg_map([ARG_STAR], [ARG_POS], [[0]], self.tuple(any_type)) + self.assert_vararg_map([ARG_STAR], [ARG_POS], [[0]], self.make_tuple(any_type)) self.assert_vararg_map( - [ARG_STAR], [ARG_POS, ARG_POS], [[0], [0]], self.tuple(any_type, any_type) + [ARG_STAR], [ARG_POS, ARG_POS], [[0], [0]], self.make_tuple(any_type, any_type) ) self.assert_vararg_map( - [ARG_STAR], [ARG_POS, ARG_OPT, ARG_OPT], [[0], [0], []], self.tuple(any_type, any_type) + [ARG_STAR], + [ARG_POS, ARG_OPT, ARG_OPT], + [[0], [0], []], + self.make_tuple(any_type, any_type), ) - def tuple(self, *args: Type) -> TupleType: + def make_tuple(self, *args: Type) -> TupleType: return TupleType(list(args), TypeFixture().std_tuple) def test_named_args(self) -> None: @@ -92,7 +93,7 @@ def test_special_cases(self) -> None: def assert_map( self, caller_kinds_: list[ArgKind | str], - callee_kinds_: list[ArgKind | Tuple[ArgKind, str]], + callee_kinds_: list[ArgKind | tuple[ArgKind, str]], expected: list[list[int]], ) -> None: caller_kinds, caller_names = expand_caller_kinds(caller_kinds_) diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py index 32586623640d..595aba49d8b7 100644 --- a/mypy/test/testmerge.py +++ b/mypy/test/testmerge.py @@ -113,6 +113,7 @@ def build(self, source: str, testcase: DataDrivenTestCase) -> BuildResult | None options.use_builtins_fixtures = True options.export_types = True options.show_traceback = True + options.allow_empty_bodies = True main_path = os.path.join(test_temp_dir, "main") with open(main_path, "w", encoding="utf8") as f: f.write(source) diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index f8990897d072..6a2d1e145251 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -32,6 +32,7 @@ def test_parser(testcase: DataDrivenTestCase) -> None: The argument contains the description of the test case. """ options = Options() + options.hide_error_codes = True if testcase.file.endswith("python310.test"): options.python_version = (3, 10) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index e4123bfdff17..1602bae6a51f 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -107,7 +107,7 @@ def test_pep561(testcase: DataDrivenTestCase) -> None: f.write(f"{s}\n") cmd_line.append(program) - cmd_line.extend(["--no-error-summary"]) + cmd_line.extend(["--no-error-summary", "--hide-error-codes"]) if python_executable != sys.executable: cmd_line.append(f"--python-executable={python_executable}") diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index a5eaea769515..6f937fee67b7 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -52,6 +52,8 @@ def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None "--no-strict-optional", "--no-silence-site-packages", "--no-error-summary", + "--hide-error-codes", + "--allow-empty-bodies", ] interpreter = python3_path mypy_cmdline.append(f"--python-version={'.'.join(map(str, PYTHON3_VERSION))}") @@ -79,7 +81,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None # Normalize paths so that the output is the same on Windows and Linux/macOS. line = line.replace(test_temp_dir + os.sep, test_temp_dir + "/") output.append(line.rstrip("\r\n")) - if returncode == 0: + if returncode == 0 and not output: # Execute the program. proc = subprocess.run( [interpreter, "-Wignore", program], cwd=test_temp_dir, capture_output=True diff --git a/mypy/test/testreports.py b/mypy/test/testreports.py index 28c4ae5638a0..a422b4bb2a7b 100644 --- a/mypy/test/testreports.py +++ b/mypy/test/testreports.py @@ -7,7 +7,7 @@ from mypy.test.helpers import Suite, assert_equal try: - import lxml # type: ignore + import lxml # type: ignore[import] except ImportError: lxml = None @@ -22,7 +22,7 @@ def test_get_line_rate(self) -> None: @pytest.mark.skipif(lxml is None, reason="Cannot import lxml. Is it installed?") def test_as_xml(self) -> None: - import lxml.etree as etree # type: ignore + import lxml.etree as etree # type: ignore[import] cobertura_package = CoberturaPackage("foobar") cobertura_package.covered_lines = 21 diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index 4f1e9d8460dd..6cfd53f09beb 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -11,7 +11,7 @@ from mypy.errors import CompileError from mypy.modulefinder import BuildSource from mypy.nodes import TypeInfo -from mypy.options import Options +from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import ( @@ -46,7 +46,7 @@ def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Opti options.semantic_analysis_only = True options.show_traceback = True options.python_version = PYTHON3_VERSION - options.enable_incomplete_features = True + options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] return options @@ -220,7 +220,7 @@ class TypeInfoMap(Dict[str, TypeInfo]): def __str__(self) -> str: a: list[str] = ["TypeInfoMap("] for x, y in sorted(self.items()): - if isinstance(x, str) and ( + if ( not x.startswith("builtins.") and not x.startswith("typing.") and not x.startswith("abc.") diff --git a/mypy/test/testsolve.py b/mypy/test/testsolve.py index 6ff328d050b3..d6c585ef4aaa 100644 --- a/mypy/test/testsolve.py +++ b/mypy/test/testsolve.py @@ -138,7 +138,7 @@ def assert_solve( assert_equal(str(actual), str(res)) def supc(self, type_var: TypeVarType, bound: Type) -> Constraint: - return Constraint(type_var.id, SUPERTYPE_OF, bound) + return Constraint(type_var, SUPERTYPE_OF, bound) def subc(self, type_var: TypeVarType, bound: Type) -> Constraint: - return Constraint(type_var.id, SUBTYPE_OF, bound) + return Constraint(type_var, SUBTYPE_OF, bound) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 7e3993252b6c..c7b576f89389 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -28,6 +28,7 @@ Options, collect_build_targets, generate_stubs, + get_sig_generators, is_blacklisted_path, is_non_library_module, mypy_options, @@ -803,7 +804,14 @@ def test_generate_c_type_stub_no_crash_for_object(self) -> None: output: list[str] = [] mod = ModuleType("module", "") # any module is fine imports: list[str] = [] - generate_c_type_stub(mod, "alias", object, output, imports) + generate_c_type_stub( + mod, + "alias", + object, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(imports, []) assert_equal(output[0], "class alias:") @@ -815,7 +823,14 @@ class TestClassVariableCls: output: list[str] = [] imports: list[str] = [] mod = ModuleType("module", "") # any module is fine - generate_c_type_stub(mod, "C", TestClassVariableCls, output, imports) + generate_c_type_stub( + mod, + "C", + TestClassVariableCls, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(imports, []) assert_equal(output, ["class C:", " x: ClassVar[int] = ..."]) @@ -826,7 +841,14 @@ class TestClass(KeyError): output: list[str] = [] imports: list[str] = [] mod = ModuleType("module, ") - generate_c_type_stub(mod, "C", TestClass, output, imports) + generate_c_type_stub( + mod, + "C", + TestClass, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(output, ["class C(KeyError): ..."]) assert_equal(imports, []) @@ -834,7 +856,14 @@ def test_generate_c_type_inheritance_same_module(self) -> None: output: list[str] = [] imports: list[str] = [] mod = ModuleType(TestBaseClass.__module__, "") - generate_c_type_stub(mod, "C", TestClass, output, imports) + generate_c_type_stub( + mod, + "C", + TestClass, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(output, ["class C(TestBaseClass): ..."]) assert_equal(imports, []) @@ -847,7 +876,14 @@ class TestClass(argparse.Action): output: list[str] = [] imports: list[str] = [] mod = ModuleType("module", "") - generate_c_type_stub(mod, "C", TestClass, output, imports) + generate_c_type_stub( + mod, + "C", + TestClass, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(output, ["class C(argparse.Action): ..."]) assert_equal(imports, ["import argparse"]) @@ -858,7 +894,14 @@ class TestClass(type): output: list[str] = [] imports: list[str] = [] mod = ModuleType("module", "") - generate_c_type_stub(mod, "C", TestClass, output, imports) + generate_c_type_stub( + mod, + "C", + TestClass, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(output, ["class C(type): ..."]) assert_equal(imports, []) @@ -873,7 +916,14 @@ def test(self, arg0: str) -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="self", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="self", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal(output, ["def test(self, arg0: int) -> Any: ..."]) assert_equal(imports, []) @@ -889,7 +939,14 @@ def test(self, arg0: str) -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="self", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="self", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal(output, ["def test(self, arg0: int) -> Any: ..."]) assert_equal(imports, []) @@ -904,11 +961,54 @@ def test(cls, arg0: str) -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="cls", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="cls", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) - assert_equal(output, ["def test(cls, *args, **kwargs) -> Any: ..."]) + assert_equal(output, ["@classmethod", "def test(cls, *args, **kwargs) -> Any: ..."]) assert_equal(imports, []) + def test_generate_c_type_classmethod_with_overloads(self) -> None: + class TestClass: + @classmethod + def test(self, arg0: str) -> None: + """ + test(cls, arg0: str) + test(cls, arg0: int) + """ + pass + + output: list[str] = [] + imports: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + generate_c_function_stub( + mod, + "test", + TestClass.test, + output, + imports, + self_var="cls", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), + ) + assert_equal( + output, + [ + "@overload", + "@classmethod", + "def test(cls, arg0: str) -> Any: ...", + "@overload", + "@classmethod", + "def test(cls, arg0: int) -> Any: ...", + ], + ) + assert_equal(imports, ["from typing import overload"]) + def test_generate_c_type_with_docstring_empty_default(self) -> None: class TestClass: def test(self, arg0: str = "") -> None: @@ -920,7 +1020,14 @@ def test(self, arg0: str = "") -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="self", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="self", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal(output, ["def test(self, arg0: str = ...) -> Any: ..."]) assert_equal(imports, []) @@ -937,7 +1044,14 @@ def test(arg0: str) -> None: output: list[str] = [] imports: list[str] = [] mod = ModuleType(self.__module__, "") - generate_c_function_stub(mod, "test", test, output, imports) + generate_c_function_stub( + mod, + "test", + test, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(output, ["def test(arg0: argparse.Action) -> Any: ..."]) assert_equal(imports, ["import argparse"]) @@ -955,7 +1069,14 @@ def test(arg0: str) -> None: output: list[str] = [] imports: list[str] = [] mod = ModuleType("argparse", "") - generate_c_function_stub(mod, "test", test, output, imports) + generate_c_function_stub( + mod, + "test", + test, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(output, ["def test(arg0: Action) -> Any: ..."]) assert_equal(imports, []) @@ -970,7 +1091,14 @@ def test(arg0: str) -> None: output: list[str] = [] imports: list[str] = [] mod = ModuleType(self.__module__, "") - generate_c_function_stub(mod, "test", test, output, imports) + generate_c_function_stub( + mod, + "test", + test, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(output, ["def test(arg0: str) -> argparse.Action: ..."]) assert_equal(imports, ["import argparse"]) @@ -987,7 +1115,14 @@ def test(arg0: str) -> None: output: list[str] = [] imports: list[str] = [] mod = ModuleType("argparse", "") - generate_c_function_stub(mod, "test", test, output, imports) + generate_c_function_stub( + mod, + "test", + test, + output, + imports, + sig_generators=get_sig_generators(parse_options([])), + ) assert_equal(output, ["def test(arg0: str) -> Action: ..."]) assert_equal(imports, []) @@ -1052,7 +1187,14 @@ def test(self, arg0: str) -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="self", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="self", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal(output, ["def test(self, arg0: List[int]) -> Any: ..."]) assert_equal(imports, []) @@ -1068,7 +1210,14 @@ def test(self, arg0: str) -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="self", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="self", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal(output, ["def test(self, arg0: Dict[str,int]) -> Any: ..."]) assert_equal(imports, []) @@ -1084,7 +1233,14 @@ def test(self, arg0: str) -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="self", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="self", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal(output, ["def test(self, arg0: Dict[str,List[int]]) -> Any: ..."]) assert_equal(imports, []) @@ -1100,7 +1256,14 @@ def test(self, arg0: str) -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="self", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="self", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal(output, ["def test(self, arg0: Dict[argparse.Action,int]) -> Any: ..."]) assert_equal(imports, ["import argparse"]) @@ -1116,7 +1279,14 @@ def test(self, arg0: str) -> None: imports: list[str] = [] mod = ModuleType(TestClass.__module__, "") generate_c_function_stub( - mod, "test", TestClass.test, output, imports, self_var="self", class_name="TestClass" + mod, + "test", + TestClass.test, + output, + imports, + self_var="self", + class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal(output, ["def test(self, arg0: Dict[str,argparse.Action]) -> Any: ..."]) assert_equal(imports, ["import argparse"]) @@ -1144,6 +1314,7 @@ def __init__(self, arg0: str) -> None: imports, self_var="self", class_name="TestClass", + sig_generators=get_sig_generators(parse_options([])), ) assert_equal( output, @@ -1153,7 +1324,7 @@ def __init__(self, arg0: str) -> None: "@overload", "def __init__(self, arg0: str, arg1: str) -> None: ...", "@overload", - "def __init__(*args, **kwargs) -> Any: ...", + "def __init__(self, *args, **kwargs) -> Any: ...", ], ) assert_equal(set(imports), {"from typing import overload"}) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index f15650811dc5..5a6904bfaaf4 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1271,6 +1271,69 @@ def test_type_var(self) -> Iterator[Case]: ) yield Case(stub="C = ParamSpec('C')", runtime="C = ParamSpec('C')", error=None) + @collect_cases + def test_metaclass_match(self) -> Iterator[Case]: + yield Case(stub="class Meta(type): ...", runtime="class Meta(type): ...", error=None) + yield Case(stub="class A0: ...", runtime="class A0: ...", error=None) + yield Case( + stub="class A1(metaclass=Meta): ...", + runtime="class A1(metaclass=Meta): ...", + error=None, + ) + yield Case(stub="class A2: ...", runtime="class A2(metaclass=Meta): ...", error="A2") + yield Case(stub="class A3(metaclass=Meta): ...", runtime="class A3: ...", error="A3") + + # Explicit `type` metaclass can always be added in any part: + yield Case( + stub="class T1(metaclass=type): ...", + runtime="class T1(metaclass=type): ...", + error=None, + ) + yield Case(stub="class T2: ...", runtime="class T2(metaclass=type): ...", error=None) + yield Case(stub="class T3(metaclass=type): ...", runtime="class T3: ...", error=None) + + # Explicit check that `_protected` names are also supported: + yield Case(stub="class _P1(type): ...", runtime="class _P1(type): ...", error=None) + yield Case(stub="class P2: ...", runtime="class P2(metaclass=_P1): ...", error="P2") + + # With inheritance: + yield Case( + stub=""" + class I1(metaclass=Meta): ... + class S1(I1): ... + """, + runtime=""" + class I1(metaclass=Meta): ... + class S1(I1): ... + """, + error=None, + ) + yield Case( + stub=""" + class I2(metaclass=Meta): ... + class S2: ... # missing inheritance + """, + runtime=""" + class I2(metaclass=Meta): ... + class S2(I2): ... + """, + error="S2", + ) + + @collect_cases + def test_metaclass_abcmeta(self) -> Iterator[Case]: + # Handling abstract metaclasses is special: + yield Case(stub="from abc import ABCMeta", runtime="from abc import ABCMeta", error=None) + yield Case( + stub="class A1(metaclass=ABCMeta): ...", + runtime="class A1(metaclass=ABCMeta): ...", + error=None, + ) + # Stubs cannot miss abstract metaclass: + yield Case(stub="class A2: ...", runtime="class A2(metaclass=ABCMeta): ...", error="A2") + # But, stubs can add extra abstract metaclass, this might be a typing hack: + yield Case(stub="class A3(metaclass=ABCMeta): ...", runtime="class A3: ...", error=None) + @collect_cases def test_abstract_methods(self) -> Iterator[Case]: yield Case( @@ -1319,6 +1382,7 @@ def some(self) -> None: ... @collect_cases def test_abstract_properties(self) -> Iterator[Case]: + # TODO: test abstract properties with setters yield Case( stub="from abc import abstractmethod", runtime="from abc import abstractmethod", @@ -1328,6 +1392,7 @@ def test_abstract_properties(self) -> Iterator[Case]: yield Case( stub=""" class AP1: + @property def some(self) -> int: ... """, runtime=""" @@ -1338,6 +1403,19 @@ def some(self) -> int: ... """, error="AP1.some", ) + yield Case( + stub=""" + class AP1_2: + def some(self) -> int: ... # missing `@property` decorator + """, + runtime=""" + class AP1_2: + @property + @abstractmethod + def some(self) -> int: ... + """, + error="AP1_2.some", + ) yield Case( stub=""" class AP2: @@ -1481,13 +1559,13 @@ def test_mypy_build(self) -> None: output = run_stubtest(stub="+", runtime="", options=[]) assert remove_color_code(output) == ( "error: not checking stubs due to failed mypy compile:\n{}.pyi:1: " - "error: invalid syntax\n".format(TEST_MODULE_NAME) + "error: invalid syntax [syntax]\n".format(TEST_MODULE_NAME) ) output = run_stubtest(stub="def f(): ...\ndef f(): ...", runtime="", options=[]) assert remove_color_code(output) == ( "error: not checking stubs due to mypy build errors:\n{}.pyi:2: " - 'error: Name "f" already defined on line 1\n'.format(TEST_MODULE_NAME) + 'error: Name "f" already defined on line 1 [no-redef]\n'.format(TEST_MODULE_NAME) ) def test_missing_stubs(self) -> None: diff --git a/mypy/test/testsubtypes.py b/mypy/test/testsubtypes.py index 22f48a88e879..c76a34ff00d7 100644 --- a/mypy/test/testsubtypes.py +++ b/mypy/test/testsubtypes.py @@ -273,6 +273,22 @@ def test_type_var_tuple_with_prefix_suffix(self) -> None: Instance(self.fx.gvi, [self.fx.a, UnpackType(self.fx.ss), self.fx.b, self.fx.c]), ) + def test_type_var_tuple_unpacked_varlength_tuple(self) -> None: + self.assert_subtype( + Instance( + self.fx.gvi, + [ + UnpackType( + TupleType( + [self.fx.a, self.fx.b], + fallback=Instance(self.fx.std_tuplei, [self.fx.o]), + ) + ) + ], + ), + Instance(self.fx.gvi, [self.fx.a, self.fx.b]), + ) + def test_type_var_tuple_unpacked_tuple(self) -> None: self.assert_subtype( Instance( @@ -333,7 +349,7 @@ def test_type_var_tuple_unpacked_tuple(self) -> None: ) def test_type_var_tuple_unpacked_variable_length_tuple(self) -> None: - self.assert_strict_subtype( + self.assert_equivalent( Instance(self.fx.gvi, [self.fx.a, self.fx.a]), Instance(self.fx.gvi, [UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))]), ) diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py index 179b2f528b1e..1d3d4468444e 100644 --- a/mypy/test/testtransform.py +++ b/mypy/test/testtransform.py @@ -7,6 +7,7 @@ from mypy import build from mypy.errors import CompileError from mypy.modulefinder import BuildSource +from mypy.options import TYPE_VAR_TUPLE, UNPACK from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages, parse_options @@ -39,7 +40,7 @@ def test_transform(testcase: DataDrivenTestCase) -> None: options = parse_options(src, testcase, 1) options.use_builtins_fixtures = True options.semantic_analysis_only = True - options.enable_incomplete_features = True + options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] options.show_traceback = True result = build.build( sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir diff --git a/mypy/test/testtypegen.py b/mypy/test/testtypegen.py index 48e1695d0278..db155a337980 100644 --- a/mypy/test/testtypegen.py +++ b/mypy/test/testtypegen.py @@ -34,6 +34,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: options.show_traceback = True options.export_types = True options.preserve_asts = True + options.allow_empty_bodies = True result = build.build( sources=[BuildSource("main", None, src)], options=options, @@ -53,7 +54,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: # Filter nodes that should be included in the output. keys = [] for node in nodes: - if node.line is not None and node.line != -1 and map[node]: + if node.line != -1 and map[node]: if ignore_node(node) or node in ignored: continue if re.match(mask, short_type(node)) or ( diff --git a/mypy/test/typefixture.py b/mypy/test/typefixture.py index a78ad6e6f51b..380da909893a 100644 --- a/mypy/test/typefixture.py +++ b/mypy/test/typefixture.py @@ -66,6 +66,7 @@ def make_type_var_tuple(name: str, id: int, upper_bound: Type) -> TypeVarTupleTy self.s1 = make_type_var("S", 1, [], self.o, variance) # S`1 (type variable) self.sf = make_type_var("S", -2, [], self.o, variance) # S`-2 (type variable) self.sf1 = make_type_var("S", -1, [], self.o, variance) # S`-1 (type variable) + self.u = make_type_var("U", 3, [], self.o, variance) # U`3 (type variable) self.ts = make_type_var_tuple("Ts", 1, self.o) # Ts`1 (type var tuple) self.ss = make_type_var_tuple("Ss", 2, self.o) # Ss`2 (type var tuple) diff --git a/mypy/treetransform.py b/mypy/treetransform.py index ca50afde7556..c863db6b3dd5 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -49,6 +49,7 @@ LambdaExpr, ListComprehension, ListExpr, + MatchStmt, MemberExpr, MypyFile, NamedTupleExpr, @@ -90,6 +91,17 @@ YieldExpr, YieldFromExpr, ) +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + Pattern, + SequencePattern, + SingletonPattern, + StarredPattern, + ValuePattern, +) from mypy.traverser import TraverserVisitor from mypy.types import FunctionLike, ProperType, Type from mypy.util import replace_object_state @@ -361,7 +373,7 @@ def visit_raise_stmt(self, node: RaiseStmt) -> RaiseStmt: return RaiseStmt(self.optional_expr(node.expr), self.optional_expr(node.from_expr)) def visit_try_stmt(self, node: TryStmt) -> TryStmt: - return TryStmt( + new = TryStmt( self.block(node.body), self.optional_names(node.vars), self.optional_expressions(node.types), @@ -369,6 +381,8 @@ def visit_try_stmt(self, node: TryStmt) -> TryStmt: self.optional_block(node.else_body), self.optional_block(node.finally_body), ) + new.is_star = node.is_star + return new def visit_with_stmt(self, node: WithStmt) -> WithStmt: new = WithStmt( @@ -381,6 +395,52 @@ def visit_with_stmt(self, node: WithStmt) -> WithStmt: new.analyzed_types = [self.type(typ) for typ in node.analyzed_types] return new + def visit_as_pattern(self, p: AsPattern) -> AsPattern: + return AsPattern( + pattern=self.pattern(p.pattern) if p.pattern is not None else None, + name=self.duplicate_name(p.name) if p.name is not None else None, + ) + + def visit_or_pattern(self, p: OrPattern) -> OrPattern: + return OrPattern([self.pattern(pat) for pat in p.patterns]) + + def visit_value_pattern(self, p: ValuePattern) -> ValuePattern: + return ValuePattern(self.expr(p.expr)) + + def visit_singleton_pattern(self, p: SingletonPattern) -> SingletonPattern: + return SingletonPattern(p.value) + + def visit_sequence_pattern(self, p: SequencePattern) -> SequencePattern: + return SequencePattern([self.pattern(pat) for pat in p.patterns]) + + def visit_starred_pattern(self, p: StarredPattern) -> StarredPattern: + return StarredPattern(self.duplicate_name(p.capture) if p.capture is not None else None) + + def visit_mapping_pattern(self, p: MappingPattern) -> MappingPattern: + return MappingPattern( + keys=[self.expr(expr) for expr in p.keys], + values=[self.pattern(pat) for pat in p.values], + rest=self.duplicate_name(p.rest) if p.rest is not None else None, + ) + + def visit_class_pattern(self, p: ClassPattern) -> ClassPattern: + class_ref = p.class_ref.accept(self) + assert isinstance(class_ref, RefExpr) + return ClassPattern( + class_ref=class_ref, + positionals=[self.pattern(pat) for pat in p.positionals], + keyword_keys=list(p.keyword_keys), + keyword_values=[self.pattern(pat) for pat in p.keyword_values], + ) + + def visit_match_stmt(self, o: MatchStmt) -> MatchStmt: + return MatchStmt( + subject=self.expr(o.subject), + patterns=[self.pattern(p) for p in o.patterns], + guards=self.optional_expressions(o.guards), + bodies=self.blocks(o.bodies), + ) + def visit_star_expr(self, node: StarExpr) -> StarExpr: return StarExpr(node.expr) @@ -637,6 +697,12 @@ def stmt(self, stmt: Statement) -> Statement: new.set_line(stmt) return new + def pattern(self, pattern: Pattern) -> Pattern: + new = pattern.accept(self) + assert isinstance(new, Pattern) + new.set_line(pattern) + return new + # Helpers # # All the node helpers also propagate line numbers. diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 3fbef63fd50e..fe404cda0bec 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -209,7 +209,7 @@ def visit_instance(self, t: Instance) -> Type: last_known_value: LiteralType | None = None if t.last_known_value is not None: raw_last_known_value = t.last_known_value.accept(self) - assert isinstance(raw_last_known_value, LiteralType) # type: ignore + assert isinstance(raw_last_known_value, LiteralType) # type: ignore[misc] last_known_value = raw_last_known_value return Instance( typ=t.type, @@ -266,7 +266,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: def visit_literal_type(self, t: LiteralType) -> Type: fallback = t.fallback.accept(self) - assert isinstance(fallback, Instance) # type: ignore + assert isinstance(fallback, Instance) # type: ignore[misc] return LiteralType(value=t.value, fallback=fallback, line=t.line, column=t.column) def visit_union_type(self, t: UnionType) -> Type: @@ -284,7 +284,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: items: list[CallableType] = [] for item in t.items: new = item.accept(self) - assert isinstance(new, CallableType) # type: ignore + assert isinstance(new, CallableType) # type: ignore[misc] items.append(new) return Overloaded(items=items) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index ae1920e234bb..35f60f54605a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -11,7 +11,7 @@ from mypy import errorcodes as codes, message_registry, nodes from mypy.errorcodes import ErrorCode from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type -from mypy.messages import MessageBuilder, format_type_bare, quote_type_string +from mypy.messages import MessageBuilder, format_type_bare, quote_type_string, wrong_type_arg_count from mypy.nodes import ( ARG_NAMED, ARG_NAMED_OPT, @@ -38,7 +38,7 @@ check_arg_names, get_nongen_builtins, ) -from mypy.options import Options +from mypy.options import UNPACK, Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs from mypy.tvar_scope import TypeVarLikeScope @@ -138,7 +138,7 @@ def analyze_type_alias( try: type = expr_to_unanalyzed_type(node, options, api.is_stub_file) except TypeTranslationError: - api.fail("Invalid type alias: expression is not a valid type", node) + api.fail("Invalid type alias: expression is not a valid type", node, code=codes.VALID_TYPE) return None analyzer = TypeAnalyser( api, @@ -281,11 +281,13 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) tvar_def = self.tvar_scope.get_binding(sym) if isinstance(sym.node, ParamSpecExpr): if tvar_def is None: - self.fail(f'ParamSpec "{t.name}" is unbound', t) + self.fail(f'ParamSpec "{t.name}" is unbound', t, code=codes.VALID_TYPE) return AnyType(TypeOfAny.from_error) assert isinstance(tvar_def, ParamSpecType) if len(t.args) > 0: - self.fail(f'ParamSpec "{t.name}" used with arguments', t) + self.fail( + f'ParamSpec "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) # Change the line number return ParamSpecType( tvar_def.name, @@ -298,15 +300,17 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) ) if isinstance(sym.node, TypeVarExpr) and tvar_def is not None and self.defining_alias: self.fail( - 'Can\'t use bound type variable "{}"' - " to define generic alias".format(t.name), + f'Can\'t use bound type variable "{t.name}" to define generic alias', t, + code=codes.VALID_TYPE, ) return AnyType(TypeOfAny.from_error) if isinstance(sym.node, TypeVarExpr) and tvar_def is not None: assert isinstance(tvar_def, TypeVarType) if len(t.args) > 0: - self.fail(f'Type variable "{t.name}" used with arguments', t) + self.fail( + f'Type variable "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) # Change the line number return TypeVarType( tvar_def.name, @@ -322,18 +326,20 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) tvar_def is not None and self.defining_alias ): self.fail( - 'Can\'t use bound type variable "{}"' - " to define generic alias".format(t.name), + f'Can\'t use bound type variable "{t.name}" to define generic alias', t, + code=codes.VALID_TYPE, ) return AnyType(TypeOfAny.from_error) if isinstance(sym.node, TypeVarTupleExpr): if tvar_def is None: - self.fail(f'TypeVarTuple "{t.name}" is unbound', t) + self.fail(f'TypeVarTuple "{t.name}" is unbound', t, code=codes.VALID_TYPE) return AnyType(TypeOfAny.from_error) assert isinstance(tvar_def, TypeVarTupleType) if len(t.args) > 0: - self.fail(f'Type variable "{t.name}" used with arguments', t) + self.fail( + f'Type variable "{t.name}" used with arguments', t, code=codes.VALID_TYPE + ) # Change the line number return TypeVarTupleType( tvar_def.name, @@ -396,24 +402,28 @@ def cannot_resolve_type(self, t: UnboundType) -> None: # need access to MessageBuilder here. Also move the similar # message generation logic in semanal.py. self.api.fail(f'Cannot resolve name "{t.name}" (possible cyclic definition)', t) - if self.options.enable_recursive_aliases and self.api.is_func_scope(): + if not self.options.disable_recursive_aliases and self.api.is_func_scope(): self.note("Recursive types are not allowed at function scope", t) def apply_concatenate_operator(self, t: UnboundType) -> Type: if len(t.args) == 0: - self.api.fail("Concatenate needs type arguments", t) + self.api.fail("Concatenate needs type arguments", t, code=codes.VALID_TYPE) return AnyType(TypeOfAny.from_error) # last argument has to be ParamSpec ps = self.anal_type(t.args[-1], allow_param_spec=True) if not isinstance(ps, ParamSpecType): - self.api.fail("The last parameter to Concatenate needs to be a ParamSpec", t) + self.api.fail( + "The last parameter to Concatenate needs to be a ParamSpec", + t, + code=codes.VALID_TYPE, + ) return AnyType(TypeOfAny.from_error) # TODO: this may not work well with aliases, if those worked. # Those should be special-cased. elif ps.prefix.arg_types: - self.api.fail("Nested Concatenates are invalid", t) + self.api.fail("Nested Concatenates are invalid", t, code=codes.VALID_TYPE) args = self.anal_array(t.args[:-1]) pre = ps.prefix @@ -437,7 +447,9 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ return AnyType(TypeOfAny.explicit) elif fullname in FINAL_TYPE_NAMES: self.fail( - "Final can be only used as an outermost qualifier in a variable annotation", t + "Final can be only used as an outermost qualifier in a variable annotation", + t, + code=codes.VALID_TYPE, ) return AnyType(TypeOfAny.from_error) elif fullname == "typing.Tuple" or ( @@ -468,7 +480,9 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ return UnionType.make_union(items) elif fullname == "typing.Optional": if len(t.args) != 1: - self.fail("Optional[...] must have exactly one type argument", t) + self.fail( + "Optional[...] must have exactly one type argument", t, code=codes.VALID_TYPE + ) return AnyType(TypeOfAny.from_error) item = self.anal_type(t.args[0]) return make_optional_type(item) @@ -488,19 +502,25 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ return None if len(t.args) != 1: type_str = "Type[...]" if fullname == "typing.Type" else "type[...]" - self.fail(type_str + " must have exactly one type argument", t) + self.fail( + type_str + " must have exactly one type argument", t, code=codes.VALID_TYPE + ) item = self.anal_type(t.args[0]) if bad_type_type_item(item): - self.fail("Type[...] can't contain another Type[...]", t) + self.fail("Type[...] can't contain another Type[...]", t, code=codes.VALID_TYPE) item = AnyType(TypeOfAny.from_error) return TypeType.make_normalized(item, line=t.line, column=t.column) elif fullname == "typing.ClassVar": if self.nesting_level > 0: - self.fail("Invalid type: ClassVar nested inside other type", t) + self.fail( + "Invalid type: ClassVar nested inside other type", t, code=codes.VALID_TYPE + ) if len(t.args) == 0: return AnyType(TypeOfAny.from_omitted_generics, line=t.line, column=t.column) if len(t.args) != 1: - self.fail("ClassVar[...] must have at most one type argument", t) + self.fail( + "ClassVar[...] must have at most one type argument", t, code=codes.VALID_TYPE + ) return AnyType(TypeOfAny.from_error) return self.anal_type(t.args[0]) elif fullname in NEVER_NAMES: @@ -513,32 +533,46 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ "Annotated[...] must have exactly one type argument" " and at least one annotation", t, + code=codes.VALID_TYPE, ) return AnyType(TypeOfAny.from_error) return self.anal_type(t.args[0]) elif fullname in ("typing_extensions.Required", "typing.Required"): if not self.allow_required: - self.fail("Required[] can be only used in a TypedDict definition", t) + self.fail( + "Required[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) return AnyType(TypeOfAny.from_error) if len(t.args) != 1: - self.fail("Required[] must have exactly one type argument", t) + self.fail( + "Required[] must have exactly one type argument", t, code=codes.VALID_TYPE + ) return AnyType(TypeOfAny.from_error) return RequiredType(self.anal_type(t.args[0]), required=True) elif fullname in ("typing_extensions.NotRequired", "typing.NotRequired"): if not self.allow_required: - self.fail("NotRequired[] can be only used in a TypedDict definition", t) + self.fail( + "NotRequired[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) return AnyType(TypeOfAny.from_error) if len(t.args) != 1: - self.fail("NotRequired[] must have exactly one type argument", t) + self.fail( + "NotRequired[] must have exactly one type argument", t, code=codes.VALID_TYPE + ) return AnyType(TypeOfAny.from_error) return RequiredType(self.anal_type(t.args[0]), required=False) elif self.anal_type_guard_arg(t, fullname) is not None: # In most contexts, TypeGuard[...] acts as an alias for bool (ignoring its args) return self.named_type("builtins.bool") elif fullname in ("typing.Unpack", "typing_extensions.Unpack"): - # We don't want people to try to use this yet. - if not self.options.enable_incomplete_features: - self.fail('"Unpack" is not supported by mypy yet', t) + if not self.api.incomplete_feature_enabled(UNPACK, t): + return AnyType(TypeOfAny.from_error) + if len(t.args) != 1: + self.fail("Unpack[...] requires exactly one type argument", t) return AnyType(TypeOfAny.from_error) return UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) return None @@ -613,21 +647,39 @@ def analyze_type_with_type_info( # The class has a Tuple[...] base class so it will be # represented as a tuple type. if info.special_alias: - return TypeAliasType(info.special_alias, self.anal_array(args)) + return expand_type_alias( + info.special_alias, + self.anal_array(args), + self.fail, + False, + ctx, + use_standard_error=True, + ) return tup.copy_modified(items=self.anal_array(tup.items), fallback=instance) td = info.typeddict_type if td is not None: # The class has a TypedDict[...] base class so it will be # represented as a typeddict type. if info.special_alias: - return TypeAliasType(info.special_alias, self.anal_array(args)) + return expand_type_alias( + info.special_alias, + self.anal_array(args), + self.fail, + False, + ctx, + use_standard_error=True, + ) # Create a named TypedDictType return td.copy_modified( item_types=self.anal_array(list(td.items.values())), fallback=instance ) if info.fullname == "types.NoneType": - self.fail("NoneType should not be used as a type, please use None instead", ctx) + self.fail( + "NoneType should not be used as a type, please use None instead", + ctx, + code=codes.VALID_TYPE, + ) return NoneType(ctx.line, ctx.column) return instance @@ -680,7 +732,7 @@ def analyze_unbound_type_without_type_info( msg = message_registry.INVALID_TYPE_RAW_ENUM_VALUE.format( base_enum_short_name, value ) - self.fail(msg, t) + self.fail(msg.value, t, code=msg.code) return AnyType(TypeOfAny.from_error) return LiteralType( value=value, @@ -709,8 +761,8 @@ def analyze_unbound_type_without_type_info( else: notes.append('Perhaps you need "Callable[...]" or a callback protocol?') elif isinstance(sym.node, MypyFile): - # TODO: suggest a protocol when supported. message = 'Module "{}" is not valid as a type' + notes.append("Perhaps you meant to use a protocol matching the module structure?") elif unbound_tvar: message = 'Type variable "{}" is unbound' short = name.split(".")[-1] @@ -763,12 +815,14 @@ def visit_type_list(self, t: TypeList) -> Type: else: return AnyType(TypeOfAny.from_error) else: - self.fail('Bracketed expression "[...]" is not valid as a type', t) + self.fail( + 'Bracketed expression "[...]" is not valid as a type', t, code=codes.VALID_TYPE + ) self.note('Did you mean "List[...]"?', t) return AnyType(TypeOfAny.from_error) def visit_callable_argument(self, t: CallableArgument) -> Type: - self.fail("Invalid type", t) + self.fail("Invalid type", t, code=codes.VALID_TYPE) return AnyType(TypeOfAny.from_error) def visit_instance(self, t: Instance) -> Type: @@ -831,24 +885,38 @@ def anal_type_guard(self, t: Type) -> Type | None: def anal_type_guard_arg(self, t: UnboundType, fullname: str) -> Type | None: if fullname in ("typing_extensions.TypeGuard", "typing.TypeGuard"): if len(t.args) != 1: - self.fail("TypeGuard must have exactly one type argument", t) + self.fail( + "TypeGuard must have exactly one type argument", t, code=codes.VALID_TYPE + ) return AnyType(TypeOfAny.from_error) return self.anal_type(t.args[0]) return None def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type: """Analyze signature argument type for *args and **kwargs argument.""" - # TODO: Check that suffix and kind match if isinstance(t, UnboundType) and t.name and "." in t.name and not t.args: components = t.name.split(".") - sym = self.lookup_qualified(".".join(components[:-1]), t) + tvar_name = ".".join(components[:-1]) + sym = self.lookup_qualified(tvar_name, t) if sym is not None and isinstance(sym.node, ParamSpecExpr): tvar_def = self.tvar_scope.get_binding(sym) if isinstance(tvar_def, ParamSpecType): if kind == ARG_STAR: make_paramspec = paramspec_args + if components[-1] != "args": + self.fail( + f'Use "{tvar_name}.args" for variadic "*" parameter', + t, + code=codes.VALID_TYPE, + ) elif kind == ARG_STAR2: make_paramspec = paramspec_kwargs + if components[-1] != "kwargs": + self.fail( + f'Use "{tvar_name}.kwargs" for variadic "**" parameter', + t, + code=codes.VALID_TYPE, + ) else: assert False, kind return make_paramspec( @@ -962,7 +1030,7 @@ def visit_union_type(self, t: UnionType) -> Type: and not self.always_allow_new_syntax and not self.options.python_version >= (3, 10) ): - self.fail("X | Y syntax for unions requires Python 3.10", t) + self.fail("X | Y syntax for unions requires Python 3.10", t, code=codes.SYNTAX) return UnionType(self.anal_array(t.items), t.line) def visit_partial_type(self, t: PartialType) -> Type: @@ -1092,6 +1160,7 @@ def analyze_callable_type(self, t: UnboundType) -> Type: "The first argument to Callable must be a " 'list of types, parameter specification, or "..."', t, + code=codes.VALID_TYPE, ) self.note( "See https://mypy.readthedocs.io/en/stable/kinds_of_types.html#callable-types-and-lambdas", # noqa: E501 @@ -1145,7 +1214,7 @@ def analyze_callable_args( def analyze_literal_type(self, t: UnboundType) -> Type: if len(t.args) == 0: - self.fail("Literal[...] must have at least one parameter", t) + self.fail("Literal[...] must have at least one parameter", t, code=codes.VALID_TYPE) return AnyType(TypeOfAny.from_error) output: list[Type] = [] @@ -1196,7 +1265,11 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> list[Type] # TODO: Once we start adding support for enums, make sure we report a custom # error for case 2 as well. if arg.type_of_any not in (TypeOfAny.from_error, TypeOfAny.special_form): - self.fail(f'Parameter {idx} of Literal[...] cannot be of type "Any"', ctx) + self.fail( + f'Parameter {idx} of Literal[...] cannot be of type "Any"', + ctx, + code=codes.VALID_TYPE, + ) return None elif isinstance(arg, RawExpressionType): # A raw literal. Convert it directly into a literal if we can. @@ -1206,7 +1279,7 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> list[Type] msg = f'Parameter {idx} of Literal[...] cannot be of type "{name}"' else: msg = "Invalid type: Literal[...] cannot contain arbitrary expressions" - self.fail(msg, ctx) + self.fail(msg, ctx, code=codes.VALID_TYPE) # Note: we deliberately ignore arg.note here: the extra info might normally be # helpful, but it generally won't make sense in the context of a Literal[...]. return None @@ -1230,7 +1303,7 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> list[Type] out.extend(union_result) return out else: - self.fail(f"Parameter {idx} of Literal[...] is invalid", ctx) + self.fail(f"Parameter {idx} of Literal[...] is invalid", ctx, code=codes.VALID_TYPE) return None def analyze_type(self, t: Type) -> Type: @@ -1277,25 +1350,29 @@ def bind_function_type_variables( ) -> Sequence[TypeVarLikeType]: """Find the type variables of the function type and bind them in our tvar_scope""" if fun_type.variables: + defs = [] for var in fun_type.variables: var_node = self.lookup_qualified(var.name, defn) assert var_node, "Binding for function type variable not found within function" var_expr = var_node.node assert isinstance(var_expr, TypeVarLikeExpr) - self.tvar_scope.bind_new(var.name, var_expr) - return fun_type.variables + binding = self.tvar_scope.bind_new(var.name, var_expr) + defs.append(binding) + return defs typevars = self.infer_type_variables(fun_type) # Do not define a new type variable if already defined in scope. typevars = [ (name, tvar) for name, tvar in typevars if not self.is_defined_type_var(name, defn) ] - defs: list[TypeVarLikeType] = [] + defs = [] for name, tvar in typevars: if not self.tvar_scope.allow_binding(tvar.fullname): - self.fail(f'Type variable "{name}" is bound by an outer class', defn) - self.tvar_scope.bind_new(name, tvar) - binding = self.tvar_scope.get_binding(tvar.fullname) - assert binding is not None + self.fail( + f'Type variable "{name}" is bound by an outer class', + defn, + code=codes.VALID_TYPE, + ) + binding = self.tvar_scope.bind_new(name, tvar) defs.append(binding) return defs @@ -1312,7 +1389,7 @@ def anal_array( res: list[Type] = [] for t in a: res.append(self.anal_type(t, nested, allow_param_spec=allow_param_spec)) - return res + return self.check_unpacks_in_list(res) def anal_type(self, t: Type, nested: bool = True, *, allow_param_spec: bool = False) -> Type: if nested: @@ -1331,10 +1408,12 @@ def anal_type(self, t: Type, nested: bool = True, *, allow_param_spec: bool = Fa and analyzed.flavor == ParamSpecFlavor.BARE ): if analyzed.prefix.arg_types: - self.fail("Invalid location for Concatenate", t) + self.fail("Invalid location for Concatenate", t, code=codes.VALID_TYPE) self.note("You can use Concatenate as the first argument to Callable", t) else: - self.fail(f'Invalid location for ParamSpec "{analyzed.name}"', t) + self.fail( + f'Invalid location for ParamSpec "{analyzed.name}"', t, code=codes.VALID_TYPE + ) self.note( "You can use ParamSpec as the first argument to Callable, e.g., " "'Callable[{}, int]'".format(analyzed.name), @@ -1369,10 +1448,30 @@ def named_type( node = self.lookup_fqn_func(fully_qualified_name) assert isinstance(node.node, TypeInfo) any_type = AnyType(TypeOfAny.special_form) + if args is not None: + args = self.check_unpacks_in_list(args) return Instance( node.node, args or [any_type] * len(node.node.defn.type_vars), line=line, column=column ) + def check_unpacks_in_list(self, items: list[Type]) -> list[Type]: + new_items: list[Type] = [] + num_unpacks = 0 + final_unpack = None + for item in items: + if isinstance(item, UnpackType): + if not num_unpacks: + new_items.append(item) + num_unpacks += 1 + final_unpack = item + else: + new_items.append(item) + + if num_unpacks > 1: + assert final_unpack is not None + self.fail("More than one Unpack in a type is not allowed", final_unpack) + return new_items + def tuple_type(self, items: list[Type]) -> TupleType: any_type = AnyType(TypeOfAny.special_form) return TupleType(items, fallback=self.named_type("builtins.tuple", [any_type])) @@ -1473,16 +1572,11 @@ def fix_instance( t.args = (any_type,) * len(t.type.type_vars) return # Invalid number of type parameters. - n = len(t.type.type_vars) - s = f"{n} type arguments" - if n == 0: - s = "no type arguments" - elif n == 1: - s = "1 type argument" - act = str(len(t.args)) - if act == "0": - act = "none" - fail(f'"{t.type.name}" expects {s}, but {act} given', t, code=codes.TYPE_ARG) + fail( + wrong_type_arg_count(len(t.type.type_vars), str(len(t.args)), t.type.name), + t, + code=codes.TYPE_ARG, + ) # Construct the correct number of type arguments, as # otherwise the type checker may crash as it expects # things to be right. @@ -1499,6 +1593,7 @@ def expand_type_alias( *, unexpanded_type: Type | None = None, disallow_any: bool = False, + use_standard_error: bool = False, ) -> Type: """Expand a (generic) type alias target following the rules outlined in TypeAlias docstring. @@ -1540,13 +1635,19 @@ def expand_type_alias( tp.column = ctx.column return tp if act_len != exp_len: - fail(f"Bad number of arguments for type alias, expected: {exp_len}, given: {act_len}", ctx) + if use_standard_error: + # This is used if type alias is an internal representation of another type, + # for example a generic TypedDict or NamedTuple. + msg = wrong_type_arg_count(exp_len, str(act_len), node.name) + else: + msg = f"Bad number of arguments for type alias, expected: {exp_len}, given: {act_len}" + fail(msg, ctx, code=codes.TYPE_ARG) return set_any_tvars(node, ctx.line, ctx.column, from_error=True) typ = TypeAliasType(node, args, ctx.line, ctx.column) assert typ.alias is not None # HACK: Implement FlexibleAlias[T, typ] by expanding it to typ here. if ( - isinstance(typ.alias.target, Instance) # type: ignore + isinstance(typ.alias.target, Instance) # type: ignore[misc] and typ.alias.target.type.fullname == "mypy_extensions.FlexibleAlias" ): exp = get_proper_type(typ) diff --git a/mypy/typeops.py b/mypy/typeops.py index b1aa2a189a20..5b29dc71991b 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -8,7 +8,7 @@ from __future__ import annotations import itertools -from typing import Any, Iterable, List, Sequence, Type as TypingType, TypeVar, cast +from typing import Any, Iterable, List, Sequence, TypeVar, cast from mypy.copytype import copy_type from mypy.expandtype import expand_type, expand_type_by_instance @@ -45,6 +45,7 @@ TupleType, Type, TypeAliasType, + TypedDictType, TypeOfAny, TypeQuery, TypeType, @@ -70,13 +71,13 @@ def is_recursive_pair(s: Type, t: Type) -> bool: """ if isinstance(s, TypeAliasType) and s.is_recursive: return ( - isinstance(get_proper_type(t), Instance) + isinstance(get_proper_type(t), (Instance, UnionType)) or isinstance(t, TypeAliasType) and t.is_recursive ) if isinstance(t, TypeAliasType) and t.is_recursive: return ( - isinstance(get_proper_type(s), Instance) + isinstance(get_proper_type(s), (Instance, UnionType)) or isinstance(s, TypeAliasType) and s.is_recursive ) @@ -104,7 +105,16 @@ def tuple_fallback(typ: TupleType) -> Instance: raise NotImplementedError else: items.append(item) - return Instance(info, [join_type_list(items)]) + return Instance(info, [join_type_list(items)], extra_attrs=typ.partial_fallback.extra_attrs) + + +def get_self_type(func: CallableType, default_self: Instance | TupleType) -> Type | None: + if isinstance(get_proper_type(func.ret_type), UninhabitedType): + return func.ret_type + elif func.arg_types and func.arg_types[0] != default_self and func.arg_kinds[0] == ARG_POS: + return func.arg_types[0] + else: + return None def type_object_type_from_function( @@ -117,14 +127,7 @@ def type_object_type_from_function( # classes such as subprocess.Popen. default_self = fill_typevars(info) if not is_new and not info.is_newtype: - orig_self_types = [ - ( - it.arg_types[0] - if it.arg_types and it.arg_types[0] != default_self and it.arg_kinds[0] == ARG_POS - else None - ) - for it in signature.items - ] + orig_self_types = [get_self_type(it, default_self) for it in signature.items] else: orig_self_types = [None] * len(signature.items) @@ -177,7 +180,7 @@ def class_callable( default_ret_type = fill_typevars(info) explicit_type = init_ret_type if is_new else orig_self_type if ( - isinstance(explicit_type, (Instance, TupleType)) + isinstance(explicit_type, (Instance, TupleType, UninhabitedType)) # We have to skip protocols, because it can be a subtype of a return type # by accident. Like `Hashable` is a subtype of `object`. See #11799 and isinstance(default_ret_type, Instance) @@ -460,7 +463,20 @@ def make_simplified_union( ): simplified_set = try_contracting_literals_in_union(simplified_set) - return get_proper_type(UnionType.make_union(simplified_set, line, column)) + result = get_proper_type(UnionType.make_union(simplified_set, line, column)) + + # Step 4: At last, we erase any (inconsistent) extra attributes on instances. + extra_attrs_set = set() + for item in items: + instance = try_getting_instance_fallback(item) + if instance and instance.extra_attrs: + extra_attrs_set.add(instance.extra_attrs) + + fallback = try_getting_instance_fallback(result) + if len(extra_attrs_set) > 1 and fallback: + fallback.extra_attrs = None + + return result def _remove_redundant_union_items(items: list[Type], keep_erased: bool) -> list[Type]: @@ -515,7 +531,7 @@ def _remove_redundant_union_items(items: list[Type], keep_erased: bool) -> list[ continue # actual redundancy checks (XXX?) if is_redundant_literal_instance(proper_item, proper_tj) and is_proper_subtype( - tj, item, keep_erased_types=keep_erased + tj, item, keep_erased_types=keep_erased, ignore_promotions=True ): # We found a redundant item in the union. removed.add(j) @@ -741,7 +757,7 @@ def try_getting_int_literals_from_type(typ: Type) -> list[int] | None: def try_getting_literals_from_type( - typ: Type, target_literal_type: TypingType[T], target_fullname: str + typ: Type, target_literal_type: type[T], target_fullname: str ) -> list[T] | None: """If the given expression or type corresponds to a Literal or union of Literals where the underlying values correspond to the given @@ -982,3 +998,21 @@ def separate_union_literals(t: UnionType) -> tuple[Sequence[LiteralType], Sequen union_items.append(item) return literal_items, union_items + + +def try_getting_instance_fallback(typ: Type) -> Instance | None: + """Returns the Instance fallback for this type if one exists or None.""" + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return typ + elif isinstance(typ, TupleType): + return typ.partial_fallback + elif isinstance(typ, TypedDictType): + return typ.fallback + elif isinstance(typ, FunctionLike): + return typ.fallback + elif isinstance(typ, LiteralType): + return typ.fallback + elif isinstance(typ, TypeVarType): + return try_getting_instance_fallback(typ.upper_bound) + return None diff --git a/mypy/types.py b/mypy/types.py index cfb6c62de147..e322cf02505f 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -11,12 +11,13 @@ Dict, Iterable, NamedTuple, + NewType, Sequence, TypeVar, Union, cast, ) -from typing_extensions import Final, TypeAlias as _TypeAlias, overload +from typing_extensions import Final, TypeAlias as _TypeAlias, TypeGuard, overload import mypy.nodes from mypy.bogus_type import Bogus @@ -516,27 +517,35 @@ def __init__( @staticmethod def new_unification_variable(old: TypeVarType) -> TypeVarType: new_id = TypeVarId.new(meta_level=1) + return old.copy_modified(id=new_id) + + def copy_modified( + self, + values: Bogus[list[Type]] = _dummy, + upper_bound: Bogus[Type] = _dummy, + id: Bogus[TypeVarId | int] = _dummy, + ) -> TypeVarType: return TypeVarType( - old.name, - old.fullname, - new_id, - old.values, - old.upper_bound, - old.variance, - old.line, - old.column, + self.name, + self.fullname, + self.id if id is _dummy else id, + self.values if values is _dummy else values, + self.upper_bound if upper_bound is _dummy else upper_bound, + self.variance, + self.line, + self.column, ) def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_var(self) def __hash__(self) -> int: - return hash(self.id) + return hash((self.id, self.upper_bound)) def __eq__(self, other: object) -> bool: if not isinstance(other, TypeVarType): return NotImplemented - return self.id == other.id + return self.id == other.id and self.upper_bound == other.upper_bound def serialize(self) -> JsonDict: assert not self.id.is_meta_var() @@ -615,16 +624,7 @@ def __init__( @staticmethod def new_unification_variable(old: ParamSpecType) -> ParamSpecType: new_id = TypeVarId.new(meta_level=1) - return ParamSpecType( - old.name, - old.fullname, - new_id, - old.flavor, - old.upper_bound, - line=old.line, - column=old.column, - prefix=old.prefix, - ) + return old.copy_modified(id=new_id) def with_flavor(self, flavor: int) -> ParamSpecType: return ParamSpecType( @@ -736,8 +736,16 @@ def __eq__(self, other: object) -> bool: @staticmethod def new_unification_variable(old: TypeVarTupleType) -> TypeVarTupleType: new_id = TypeVarId.new(meta_level=1) + return old.copy_modified(id=new_id) + + def copy_modified(self, id: Bogus[TypeVarId | int] = _dummy) -> TypeVarTupleType: return TypeVarTupleType( - old.name, old.fullname, new_id, old.upper_bound, line=old.line, column=old.column + self.name, + self.fullname, + self.id if id is _dummy else id, + self.upper_bound, + line=self.line, + column=self.column, ) @@ -868,7 +876,7 @@ def __init__( def accept(self, visitor: TypeVisitor[T]) -> T: assert isinstance(visitor, SyntheticTypeVisitor) - return visitor.visit_callable_argument(self) + return cast(T, visitor.visit_callable_argument(self)) def serialize(self) -> JsonDict: assert False, "Synthetic types don't serialize" @@ -893,7 +901,7 @@ def __init__(self, items: list[Type], line: int = -1, column: int = -1) -> None: def accept(self, visitor: TypeVisitor[T]) -> T: assert isinstance(visitor, SyntheticTypeVisitor) - return visitor.visit_type_list(self) + return cast(T, visitor.visit_type_list(self)) def serialize(self) -> JsonDict: assert False, "Synthetic types don't serialize" @@ -1154,38 +1162,51 @@ def deserialize(cls, data: JsonDict) -> DeletedType: NOT_READY: Final = mypy.nodes.FakeInfo("De-serialization failure: TypeInfo not fixed") +class ExtraAttrs: + """Summary of module attributes and types. + + This is used for instances of types.ModuleType, because they can have different + attributes per instance, and for type narrowing with hasattr() checks. + """ + + def __init__( + self, + attrs: dict[str, Type], + immutable: set[str] | None = None, + mod_name: str | None = None, + ) -> None: + self.attrs = attrs + if immutable is None: + immutable = set() + self.immutable = immutable + self.mod_name = mod_name + + def __hash__(self) -> int: + return hash((tuple(self.attrs.items()), tuple(sorted(self.immutable)))) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ExtraAttrs): + return NotImplemented + return self.attrs == other.attrs and self.immutable == other.immutable + + def copy(self) -> ExtraAttrs: + return ExtraAttrs(self.attrs.copy(), self.immutable.copy(), self.mod_name) + + def __repr__(self) -> str: + return f"ExtraAttrs({self.attrs!r}, {self.immutable!r}, {self.mod_name!r})" + + class Instance(ProperType): """An instance type of form C[T1, ..., Tn]. The list of type variables may be empty. - Several types has fallbacks to `Instance`. Why? - Because, for example `TupleTuple` is related to `builtins.tuple` instance. - And `FunctionLike` has `builtins.function` fallback. - This allows us to use types defined - in typeshed for our "special" and more precise types. - - We used to have this helper function to get a fallback from different types. - Note, that it might be incomplete, since it is not used and not updated. - It just illustrates the concept: - - def try_getting_instance_fallback(typ: ProperType) -> Optional[Instance]: - '''Returns the Instance fallback for this type if one exists or None.''' - if isinstance(typ, Instance): - return typ - elif isinstance(typ, TupleType): - return tuple_fallback(typ) - elif isinstance(typ, TypedDictType): - return typ.fallback - elif isinstance(typ, FunctionLike): - return typ.fallback - elif isinstance(typ, LiteralType): - return typ.fallback - return None - + Several types have fallbacks to `Instance`, because in Python everything is an object + and this concept is impossible to express without intersection types. We therefore use + fallbacks for all "non-special" (like UninhabitedType, ErasedType etc) types. """ - __slots__ = ("type", "args", "invalid", "type_ref", "last_known_value", "_hash") + __slots__ = ("type", "args", "invalid", "type_ref", "last_known_value", "_hash", "extra_attrs") def __init__( self, @@ -1195,6 +1216,7 @@ def __init__( column: int = -1, *, last_known_value: LiteralType | None = None, + extra_attrs: ExtraAttrs | None = None, ) -> None: super().__init__(line, column) self.type = typ @@ -1252,12 +1274,17 @@ def __init__( # Cached hash value self._hash = -1 + # Additional attributes defined per instance of this type. For example modules + # have different attributes per instance of types.ModuleType. This is intended + # to be "short-lived", we don't serialize it, and even don't store as variable type. + self.extra_attrs = extra_attrs + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_instance(self) def __hash__(self) -> int: if self._hash == -1: - self._hash = hash((self.type, self.args, self.last_known_value)) + self._hash = hash((self.type, self.args, self.last_known_value, self.extra_attrs)) return self._hash def __eq__(self, other: object) -> bool: @@ -1267,6 +1294,7 @@ def __eq__(self, other: object) -> bool: self.type == other.type and self.args == other.args and self.last_known_value == other.last_known_value + and self.extra_attrs == other.extra_attrs ) def serialize(self) -> JsonDict | str: @@ -1314,10 +1342,21 @@ def copy_modified( if last_known_value is not _dummy else self.last_known_value, ) + # We intentionally don't copy the extra_attrs here, so they will be erased. new.can_be_true = self.can_be_true new.can_be_false = self.can_be_false return new + def copy_with_extra_attr(self, name: str, typ: Type) -> Instance: + if self.extra_attrs: + existing_attrs = self.extra_attrs.copy() + else: + existing_attrs = ExtraAttrs({}, set(), None) + existing_attrs.attrs[name] = typ + new = self.copy_modified() + new.extra_attrs = existing_attrs + return new + def has_readable_member(self, name: str) -> bool: return self.type.has_readable_member(name) @@ -1561,6 +1600,9 @@ def __eq__(self, other: object) -> bool: return NotImplemented +CT = TypeVar("CT", bound="CallableType") + + class CallableType(FunctionLike): """Type of a non-overloaded callable object (such as function).""" @@ -1590,6 +1632,7 @@ class CallableType(FunctionLike): "type_guard", # T, if -> TypeGuard[T] (ret_type is bool in this case). "from_concatenate", # whether this callable is from a concatenate object # (this is used for error messages) + "unpack_kwargs", # Was an Unpack[...] with **kwargs used to define this callable? ) def __init__( @@ -1613,6 +1656,7 @@ def __init__( def_extras: dict[str, Any] | None = None, type_guard: Type | None = None, from_concatenate: bool = False, + unpack_kwargs: bool = False, ) -> None: super().__init__(line, column) assert len(arg_types) == len(arg_kinds) == len(arg_names) @@ -1653,9 +1697,10 @@ def __init__( else: self.def_extras = {} self.type_guard = type_guard + self.unpack_kwargs = unpack_kwargs def copy_modified( - self, + self: CT, arg_types: Bogus[Sequence[Type]] = _dummy, arg_kinds: Bogus[list[ArgKind]] = _dummy, arg_names: Bogus[list[str | None]] = _dummy, @@ -1674,8 +1719,9 @@ def copy_modified( def_extras: Bogus[dict[str, Any]] = _dummy, type_guard: Bogus[Type | None] = _dummy, from_concatenate: Bogus[bool] = _dummy, - ) -> CallableType: - return CallableType( + unpack_kwargs: Bogus[bool] = _dummy, + ) -> CT: + return type(self)( arg_types=arg_types if arg_types is not _dummy else self.arg_types, arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, arg_names=arg_names if arg_names is not _dummy else self.arg_names, @@ -1698,6 +1744,7 @@ def copy_modified( from_concatenate=( from_concatenate if from_concatenate is not _dummy else self.from_concatenate ), + unpack_kwargs=unpack_kwargs if unpack_kwargs is not _dummy else self.unpack_kwargs, ) def var_arg(self) -> FormalArgument | None: @@ -1725,7 +1772,9 @@ def is_kw_arg(self) -> bool: return ARG_STAR2 in self.arg_kinds def is_type_obj(self) -> bool: - return self.fallback.type.is_metaclass() + return self.fallback.type.is_metaclass() and not isinstance( + get_proper_type(self.ret_type), UninhabitedType + ) def type_object(self) -> mypy.nodes.TypeInfo: assert self.is_type_obj() @@ -1889,6 +1938,27 @@ def expand_param_spec( variables=[*variables, *self.variables], ) + def with_unpacked_kwargs(self) -> NormalizedCallableType: + if not self.unpack_kwargs: + return NormalizedCallableType(self.copy_modified()) + last_type = get_proper_type(self.arg_types[-1]) + assert isinstance(last_type, TypedDictType) + extra_kinds = [ + ArgKind.ARG_NAMED if name in last_type.required_keys else ArgKind.ARG_NAMED_OPT + for name in last_type.items + ] + new_arg_kinds = self.arg_kinds[:-1] + extra_kinds + new_arg_names = self.arg_names[:-1] + list(last_type.items) + new_arg_types = self.arg_types[:-1] + list(last_type.items.values()) + return NormalizedCallableType( + self.copy_modified( + arg_kinds=new_arg_kinds, + arg_names=new_arg_names, + arg_types=new_arg_types, + unpack_kwargs=False, + ) + ) + def __hash__(self) -> int: # self.is_type_obj() will fail if self.fallback.type is a FakeInfo if isinstance(self.fallback.type, FakeInfo): @@ -1904,6 +1974,7 @@ def __hash__(self) -> int: tuple(self.arg_types), tuple(self.arg_names), tuple(self.arg_kinds), + self.fallback, ) ) @@ -1917,6 +1988,7 @@ def __eq__(self, other: object) -> bool: and self.name == other.name and self.is_type_obj() == other.is_type_obj() and self.is_ellipsis_args == other.is_ellipsis_args + and self.fallback == other.fallback ) else: return NotImplemented @@ -1940,6 +2012,7 @@ def serialize(self) -> JsonDict: "def_extras": dict(self.def_extras), "type_guard": self.type_guard.serialize() if self.type_guard is not None else None, "from_concatenate": self.from_concatenate, + "unpack_kwargs": self.unpack_kwargs, } @classmethod @@ -1962,9 +2035,16 @@ def deserialize(cls, data: JsonDict) -> CallableType: deserialize_type(data["type_guard"]) if data["type_guard"] is not None else None ), from_concatenate=data["from_concatenate"], + unpack_kwargs=data["unpack_kwargs"], ) +# This is a little safety net to prevent reckless special-casing of callables +# that can potentially break Unpack[...] with **kwargs. +# TODO: use this in more places in checkexpr.py etc? +NormalizedCallableType = NewType("NormalizedCallableType", CallableType) + + class Overloaded(FunctionLike): """Overloaded function type T1, ... Tn, where each Ti is CallableType. @@ -2009,6 +2089,9 @@ def with_name(self, name: str) -> Overloaded: def get_name(self) -> str | None: return self._items[0].name + def with_unpacked_kwargs(self) -> Overloaded: + return Overloaded([i.with_unpacked_kwargs() for i in self.items]) + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_overloaded(self) @@ -2322,7 +2405,7 @@ def simple_name(self) -> str: def accept(self, visitor: TypeVisitor[T]) -> T: assert isinstance(visitor, SyntheticTypeVisitor) - return visitor.visit_raw_expression_type(self) + return cast(T, visitor.visit_raw_expression_type(self)) def serialize(self) -> JsonDict: assert False, "Synthetic types don't serialize" @@ -2445,7 +2528,7 @@ def __init__(self, type: Type, line: int = -1, column: int = -1) -> None: def accept(self, visitor: TypeVisitor[T]) -> T: assert isinstance(visitor, SyntheticTypeVisitor) - return visitor.visit_star_type(self) + return cast(T, visitor.visit_star_type(self)) def serialize(self) -> JsonDict: assert False, "Synthetic types don't serialize" @@ -2587,7 +2670,7 @@ class EllipsisType(ProperType): def accept(self, visitor: TypeVisitor[T]) -> T: assert isinstance(visitor, SyntheticTypeVisitor) - return visitor.visit_ellipsis_type(self) + return cast(T, visitor.visit_ellipsis_type(self)) def serialize(self) -> JsonDict: assert False, "Synthetic types don't serialize" @@ -2696,7 +2779,7 @@ def __init__(self, fullname: str | None, args: list[Type], line: int) -> None: def accept(self, visitor: TypeVisitor[T]) -> T: assert isinstance(visitor, SyntheticTypeVisitor) - return visitor.visit_placeholder_type(self) + return cast(T, visitor.visit_placeholder_type(self)) def serialize(self) -> str: # We should never get here since all placeholders should be replaced @@ -2917,7 +3000,10 @@ def visit_callable_type(self, t: CallableType) -> str: name = t.arg_names[i] if name: s += name + ": " - s += t.arg_types[i].accept(self) + type_str = t.arg_types[i].accept(self) + if t.arg_kinds[i] == ARG_STAR2 and t.unpack_kwargs: + type_str = f"Unpack[{type_str}]" + s += type_str if t.arg_kinds[i].is_optional(): s += " =" @@ -3091,7 +3177,7 @@ def strip_type(typ: Type) -> Type: return orig_typ -def is_named_instance(t: Type, fullnames: str | tuple[str, ...]) -> bool: +def is_named_instance(t: Type, fullnames: str | tuple[str, ...]) -> TypeGuard[Instance]: if not isinstance(fullnames, tuple): fullnames = (fullnames,) @@ -3267,6 +3353,16 @@ def is_literal_type(typ: ProperType, fallback_fullname: str, value: LiteralValue return typ.value == value +def is_self_type_like(typ: Type, *, is_classmethod: bool) -> bool: + """Does this look like a self-type annotation?""" + typ = get_proper_type(typ) + if not is_classmethod: + return isinstance(typ, TypeVarType) + if not isinstance(typ, TypeType): + return False + return isinstance(typ.item, TypeVarType) + + names: Final = globals().copy() names.pop("NOT_READY", None) deserialize_map: Final = { diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index d396ce4d0560..bd1abd204885 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -27,6 +27,7 @@ _collections_abc: 3.3- _compat_pickle: 3.1- _compression: 3.5- _csv: 2.7- +_ctypes: 2.7- _curses: 2.7- _decimal: 3.3- _dummy_thread: 3.0-3.8 diff --git a/mypy/typeshed/stdlib/_ast.pyi b/mypy/typeshed/stdlib/_ast.pyi index c68e921babd0..b7d081f6acb2 100644 --- a/mypy/typeshed/stdlib/_ast.pyi +++ b/mypy/typeshed/stdlib/_ast.pyi @@ -194,7 +194,7 @@ class Import(stmt): class ImportFrom(stmt): if sys.version_info >= (3, 10): __match_args__ = ("module", "names", "level") - module: _Identifier | None + module: str | None names: list[alias] level: int diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi new file mode 100644 index 000000000000..0ad2fcb571b8 --- /dev/null +++ b/mypy/typeshed/stdlib/_ctypes.pyi @@ -0,0 +1,29 @@ +import sys +from ctypes import _CArgObject, _PointerLike +from typing_extensions import TypeAlias + +FUNCFLAG_CDECL: int +FUNCFLAG_PYTHONAPI: int +FUNCFLAG_USE_ERRNO: int +FUNCFLAG_USE_LASTERROR: int +RTLD_GLOBAL: int +RTLD_LOCAL: int + +if sys.version_info >= (3, 11): + CTYPES_MAX_ARGCOUNT: int + +if sys.platform == "win32": + # Description, Source, HelpFile, HelpContext, scode + _COMError_Details: TypeAlias = tuple[str | None, str | None, str | None, int | None, int | None] + + class COMError(Exception): + hresult: int + text: str | None + details: _COMError_Details + + def __init__(self, hresult: int, text: str | None, details: _COMError_Details) -> None: ... + + def CopyComPointer(src: _PointerLike, dst: _PointerLike | _CArgObject) -> int: ... + + FUNCFLAG_HRESULT: int + FUNCFLAG_STDCALL: int diff --git a/mypy/typeshed/stdlib/_dummy_threading.pyi b/mypy/typeshed/stdlib/_dummy_threading.pyi index c956946c8363..8f7f5a9b994c 100644 --- a/mypy/typeshed/stdlib/_dummy_threading.pyi +++ b/mypy/typeshed/stdlib/_dummy_threading.pyi @@ -86,7 +86,6 @@ class Thread: class _DummyThread(Thread): ... class Lock: - def __init__(self) -> None: ... def __enter__(self) -> bool: ... def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None @@ -96,7 +95,6 @@ class Lock: def locked(self) -> bool: ... class _RLock: - def __init__(self) -> None: ... def __enter__(self) -> bool: ... def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None @@ -135,7 +133,6 @@ class Semaphore: class BoundedSemaphore(Semaphore): ... class Event: - def __init__(self) -> None: ... def is_set(self) -> bool: ... def set(self) -> None: ... def clear(self) -> None: ... diff --git a/mypy/typeshed/stdlib/_socket.pyi b/mypy/typeshed/stdlib/_socket.pyi index 09dbaae3dc64..b2f77893d273 100644 --- a/mypy/typeshed/stdlib/_socket.pyi +++ b/mypy/typeshed/stdlib/_socket.pyi @@ -624,7 +624,7 @@ class socket: __buffers: Iterable[ReadableBuffer], __ancdata: Iterable[_CMSGArg] = ..., __flags: int = ..., - __address: _Address = ..., + __address: _Address | None = ..., ) -> int: ... if sys.platform == "linux": def sendmsg_afalg( diff --git a/mypy/typeshed/stdlib/_threading_local.pyi b/mypy/typeshed/stdlib/_threading_local.pyi index d455ce09227e..98683dabcef8 100644 --- a/mypy/typeshed/stdlib/_threading_local.pyi +++ b/mypy/typeshed/stdlib/_threading_local.pyi @@ -14,3 +14,4 @@ class _localimpl: class local: def __getattribute__(self, name: str) -> Any: ... def __setattr__(self, name: str, value: Any) -> None: ... + def __delattr__(self, name: str) -> None: ... diff --git a/mypy/typeshed/stdlib/_typeshed/__init__.pyi b/mypy/typeshed/stdlib/_typeshed/__init__.pyi index 89ca9d81619a..b0ee1f4ad48a 100644 --- a/mypy/typeshed/stdlib/_typeshed/__init__.pyi +++ b/mypy/typeshed/stdlib/_typeshed/__init__.pyi @@ -7,7 +7,7 @@ import ctypes import mmap import pickle import sys -from collections.abc import Awaitable, Callable, Container, Iterable, Set as AbstractSet +from collections.abc import Awaitable, Callable, Iterable, Set as AbstractSet from os import PathLike from types import FrameType, TracebackType from typing import Any, AnyStr, Generic, Protocol, TypeVar, Union @@ -115,16 +115,17 @@ class SupportsItems(Protocol[_KT_co, _VT_co]): # stable class SupportsKeysAndGetItem(Protocol[_KT, _VT_co]): def keys(self) -> Iterable[_KT]: ... - def __getitem__(self, __k: _KT) -> _VT_co: ... + def __getitem__(self, __key: _KT) -> _VT_co: ... # stable -class SupportsGetItem(Container[_KT_contra], Protocol[_KT_contra, _VT_co]): - def __getitem__(self, __k: _KT_contra) -> _VT_co: ... +class SupportsGetItem(Protocol[_KT_contra, _VT_co]): + def __contains__(self, __x: object) -> bool: ... + def __getitem__(self, __key: _KT_contra) -> _VT_co: ... # stable class SupportsItemAccess(SupportsGetItem[_KT_contra, _VT], Protocol[_KT_contra, _VT]): - def __setitem__(self, __k: _KT_contra, __v: _VT) -> None: ... - def __delitem__(self, __v: _KT_contra) -> None: ... + def __setitem__(self, __key: _KT_contra, __value: _VT) -> None: ... + def __delitem__(self, __key: _KT_contra) -> None: ... StrPath: TypeAlias = str | PathLike[str] # stable BytesPath: TypeAlias = bytes | PathLike[bytes] # stable diff --git a/mypy/typeshed/stdlib/_winapi.pyi b/mypy/typeshed/stdlib/_winapi.pyi index 259293c51fd3..ddea3d67ed14 100644 --- a/mypy/typeshed/stdlib/_winapi.pyi +++ b/mypy/typeshed/stdlib/_winapi.pyi @@ -106,7 +106,7 @@ if sys.platform == "win32": WAIT_OBJECT_0: Literal[0] WAIT_TIMEOUT: Literal[258] - if sys.version_info >= (3, 11): + if sys.version_info >= (3, 10): LOCALE_NAME_INVARIANT: str LOCALE_NAME_MAX_LENGTH: int LOCALE_NAME_SYSTEM_DEFAULT: str @@ -181,7 +181,7 @@ if sys.platform == "win32": def GetVersion() -> int: ... def OpenProcess(__desired_access: int, __inherit_handle: bool, __process_id: int) -> int: ... def PeekNamedPipe(__handle: int, __size: int = ...) -> tuple[int, int] | tuple[bytes, int, int]: ... - if sys.version_info >= (3, 11): + if sys.version_info >= (3, 10): def LCMapStringEx(locale: str, flags: int, src: str) -> str: ... @overload diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi index 1b86a4e10cbb..1bdcace7d897 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -127,6 +127,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): _optionals: _ArgumentGroup _subparsers: _ArgumentGroup | None + # Note: the constructor arguments are also used in _SubParsersAction.add_parser. if sys.version_info >= (3, 9): def __init__( self, @@ -398,6 +399,10 @@ class _StoreFalseAction(_StoreConstAction): # undocumented class _AppendAction(Action): ... +# undocumented +if sys.version_info >= (3, 8): + class _ExtendAction(_AppendAction): ... + # undocumented class _AppendConstAction(Action): if sys.version_info >= (3, 11): @@ -458,8 +463,53 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): help: str | None = ..., metavar: str | tuple[str, ...] | None = ..., ) -> None: ... - # TODO: Type keyword args properly. - def add_parser(self, name: str, **kwargs: Any) -> _ArgumentParserT: ... + + # Note: `add_parser` accepts all kwargs of `ArgumentParser.__init__`. It also + # accepts its own `help` and `aliases` kwargs. + if sys.version_info >= (3, 9): + def add_parser( + self, + name: str, + *, + help: str | None = ..., + aliases: Sequence[str] = ..., + # Kwargs from ArgumentParser constructor + prog: str | None = ..., + usage: str | None = ..., + description: str | None = ..., + epilog: str | None = ..., + parents: Sequence[_ArgumentParserT] = ..., + formatter_class: _FormatterClass = ..., + prefix_chars: str = ..., + fromfile_prefix_chars: str | None = ..., + argument_default: Any = ..., + conflict_handler: str = ..., + add_help: bool = ..., + allow_abbrev: bool = ..., + exit_on_error: bool = ..., + ) -> _ArgumentParserT: ... + else: + def add_parser( + self, + name: str, + *, + help: str | None = ..., + aliases: Sequence[str] = ..., + # Kwargs from ArgumentParser constructor + prog: str | None = ..., + usage: str | None = ..., + description: str | None = ..., + epilog: str | None = ..., + parents: Sequence[_ArgumentParserT] = ..., + formatter_class: _FormatterClass = ..., + prefix_chars: str = ..., + fromfile_prefix_chars: str | None = ..., + argument_default: Any = ..., + conflict_handler: str = ..., + add_help: bool = ..., + allow_abbrev: bool = ..., + ) -> _ArgumentParserT: ... + def _get_subactions(self) -> list[Action]: ... # undocumented diff --git a/mypy/typeshed/stdlib/asyncio/mixins.pyi b/mypy/typeshed/stdlib/asyncio/mixins.pyi index 3e04f2b37518..6ebcf543e6b9 100644 --- a/mypy/typeshed/stdlib/asyncio/mixins.pyi +++ b/mypy/typeshed/stdlib/asyncio/mixins.pyi @@ -1,9 +1,9 @@ import sys import threading -from typing import NoReturn +from typing_extensions import Never _global_lock: threading.Lock class _LoopBoundMixin: if sys.version_info < (3, 11): - def __init__(self, *, loop: NoReturn = ...) -> None: ... + def __init__(self, *, loop: Never = ...) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/taskgroups.pyi b/mypy/typeshed/stdlib/asyncio/taskgroups.pyi index 9b2f15506c50..0d508c97c1f9 100644 --- a/mypy/typeshed/stdlib/asyncio/taskgroups.pyi +++ b/mypy/typeshed/stdlib/asyncio/taskgroups.pyi @@ -13,7 +13,6 @@ __all__ = ["TaskGroup"] _T = TypeVar("_T") class TaskGroup: - def __init__(self) -> None: ... async def __aenter__(self: Self) -> Self: ... async def __aexit__(self, et: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None) -> None: ... def create_task( diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index 76755f1109c3..67581eb6a5ad 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -36,6 +36,7 @@ __all__ = ( ) _T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) _T1 = TypeVar("_T1") _T2 = TypeVar("_T2") _T3 = TypeVar("_T3") @@ -265,21 +266,25 @@ else: ) -> tuple[set[Task[_T]], set[Task[_T]]]: ... async def wait_for(fut: _FutureLike[_T], timeout: float | None, *, loop: AbstractEventLoop | None = ...) -> _T: ... -class Task(Future[_T], Generic[_T]): +# mypy and pyright complain that a subclass of an invariant class shouldn't be covariant. +# While this is true in general, here it's sort-of okay to have a covariant subclass, +# since the only reason why `asyncio.Future` is invariant is the `set_result()` method, +# and `asyncio.Task.set_result()` always raises. +class Task(Future[_T_co], Generic[_T_co]): # type: ignore[type-var] if sys.version_info >= (3, 8): def __init__( self, - coro: Generator[_TaskYieldType, None, _T] | Awaitable[_T], + coro: Generator[_TaskYieldType, None, _T_co] | Awaitable[_T_co], *, loop: AbstractEventLoop = ..., name: str | None = ..., ) -> None: ... else: def __init__( - self, coro: Generator[_TaskYieldType, None, _T] | Awaitable[_T], *, loop: AbstractEventLoop = ... + self, coro: Generator[_TaskYieldType, None, _T_co] | Awaitable[_T_co], *, loop: AbstractEventLoop = ... ) -> None: ... if sys.version_info >= (3, 8): - def get_coro(self) -> Generator[_TaskYieldType, None, _T] | Awaitable[_T]: ... + def get_coro(self) -> Generator[_TaskYieldType, None, _T_co] | Awaitable[_T_co]: ... def get_name(self) -> str: ... def set_name(self, __value: object) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/transports.pyi b/mypy/typeshed/stdlib/asyncio/transports.pyi index 52937c9bcbdf..3eb3d1ae3173 100644 --- a/mypy/typeshed/stdlib/asyncio/transports.pyi +++ b/mypy/typeshed/stdlib/asyncio/transports.pyi @@ -1,14 +1,14 @@ from asyncio.events import AbstractEventLoop from asyncio.protocols import BaseProtocol -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from socket import _Address from typing import Any __all__ = ("BaseTransport", "ReadTransport", "WriteTransport", "Transport", "DatagramTransport", "SubprocessTransport") class BaseTransport: - def __init__(self, extra: Mapping[Any, Any] | None = ...) -> None: ... - def get_extra_info(self, name: Any, default: Any = ...) -> Any: ... + def __init__(self, extra: Mapping[str, Any] | None = ...) -> None: ... + def get_extra_info(self, name: str, default: Any = ...) -> Any: ... def is_closing(self) -> bool: ... def close(self) -> None: ... def set_protocol(self, protocol: BaseProtocol) -> None: ... @@ -23,8 +23,8 @@ class WriteTransport(BaseTransport): def set_write_buffer_limits(self, high: int | None = ..., low: int | None = ...) -> None: ... def get_write_buffer_size(self) -> int: ... def get_write_buffer_limits(self) -> tuple[int, int]: ... - def write(self, data: Any) -> None: ... - def writelines(self, list_of_data: list[Any]) -> None: ... + def write(self, data: bytes) -> None: ... + def writelines(self, list_of_data: Iterable[bytes]) -> None: ... def write_eof(self) -> None: ... def can_write_eof(self) -> bool: ... def abort(self) -> None: ... @@ -32,16 +32,16 @@ class WriteTransport(BaseTransport): class Transport(ReadTransport, WriteTransport): ... class DatagramTransport(BaseTransport): - def sendto(self, data: Any, addr: _Address | None = ...) -> None: ... + def sendto(self, data: bytes, addr: _Address | None = ...) -> None: ... def abort(self) -> None: ... class SubprocessTransport(BaseTransport): def get_pid(self) -> int: ... def get_returncode(self) -> int | None: ... def get_pipe_transport(self, fd: int) -> BaseTransport | None: ... - def send_signal(self, signal: int) -> int: ... + def send_signal(self, signal: int) -> None: ... def terminate(self) -> None: ... def kill(self) -> None: ... class _FlowControlMixin(Transport): - def __init__(self, extra: Mapping[Any, Any] | None = ..., loop: AbstractEventLoop | None = ...) -> None: ... + def __init__(self, extra: Mapping[str, Any] | None = ..., loop: AbstractEventLoop | None = ...) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/unix_events.pyi b/mypy/typeshed/stdlib/asyncio/unix_events.pyi index f63011a373be..19dd3ca43b95 100644 --- a/mypy/typeshed/stdlib/asyncio/unix_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/unix_events.pyi @@ -118,7 +118,6 @@ if sys.platform != "win32": if sys.version_info >= (3, 9): class PidfdChildWatcher(AbstractChildWatcher): - def __init__(self) -> None: ... def __enter__(self: Self) -> Self: ... def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None diff --git a/mypy/typeshed/stdlib/binhex.pyi b/mypy/typeshed/stdlib/binhex.pyi index 27aa379f134d..639d30d1d0de 100644 --- a/mypy/typeshed/stdlib/binhex.pyi +++ b/mypy/typeshed/stdlib/binhex.pyi @@ -10,7 +10,6 @@ LINELEN: Literal[64] RUNCHAR: Literal[b"\x90"] class FInfo: - def __init__(self) -> None: ... Type: str Creator: str Flags: int diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 6992dc8e2674..d3b3f677b370 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -266,8 +266,6 @@ class int: @overload def __pow__(self, __x: int, __modulo: None = ...) -> Any: ... @overload - def __pow__(self, __x: int, __modulo: Literal[0]) -> NoReturn: ... - @overload def __pow__(self, __x: int, __modulo: int) -> int: ... def __rpow__(self, __x: int, __mod: int | None = ...) -> Any: ... def __and__(self, __n: int) -> int: ... @@ -408,26 +406,29 @@ class complex: class _FormatMapMapping(Protocol): def __getitem__(self, __key: str) -> Any: ... +class _TranslateTable(Protocol): + def __getitem__(self, __key: int) -> str | int | None: ... + class str(Sequence[str]): @overload def __new__(cls: type[Self], object: object = ...) -> Self: ... @overload def __new__(cls: type[Self], object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... - def capitalize(self) -> str: ... - def casefold(self) -> str: ... - def center(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... + def capitalize(self) -> str: ... # type: ignore[misc] + def casefold(self) -> str: ... # type: ignore[misc] + def center(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... # type: ignore[misc] def count(self, x: str, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...) -> int: ... def encode(self, encoding: str = ..., errors: str = ...) -> bytes: ... def endswith( self, __suffix: str | tuple[str, ...], __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ... ) -> bool: ... if sys.version_info >= (3, 8): - def expandtabs(self, tabsize: SupportsIndex = ...) -> str: ... + def expandtabs(self, tabsize: SupportsIndex = ...) -> str: ... # type: ignore[misc] else: - def expandtabs(self, tabsize: int = ...) -> str: ... + def expandtabs(self, tabsize: int = ...) -> str: ... # type: ignore[misc] def find(self, __sub: str, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...) -> int: ... - def format(self, *args: object, **kwargs: object) -> str: ... + def format(self, *args: object, **kwargs: object) -> str: ... # type: ignore[misc] def format_map(self, map: _FormatMapMapping) -> str: ... def index(self, __sub: str, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...) -> int: ... def isalnum(self) -> bool: ... @@ -442,55 +443,54 @@ class str(Sequence[str]): def isspace(self) -> bool: ... def istitle(self) -> bool: ... def isupper(self) -> bool: ... - def join(self, __iterable: Iterable[str]) -> str: ... - def ljust(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... - def lower(self) -> str: ... - def lstrip(self, __chars: str | None = ...) -> str: ... - def partition(self, __sep: str) -> tuple[str, str, str]: ... - def replace(self, __old: str, __new: str, __count: SupportsIndex = ...) -> str: ... + def join(self, __iterable: Iterable[str]) -> str: ... # type: ignore[misc] + def ljust(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... # type: ignore[misc] + def lower(self) -> str: ... # type: ignore[misc] + def lstrip(self, __chars: str | None = ...) -> str: ... # type: ignore[misc] + def partition(self, __sep: str) -> tuple[str, str, str]: ... # type: ignore[misc] + def replace(self, __old: str, __new: str, __count: SupportsIndex = ...) -> str: ... # type: ignore[misc] if sys.version_info >= (3, 9): - def removeprefix(self, __prefix: str) -> str: ... - def removesuffix(self, __suffix: str) -> str: ... + def removeprefix(self, __prefix: str) -> str: ... # type: ignore[misc] + def removesuffix(self, __suffix: str) -> str: ... # type: ignore[misc] def rfind(self, __sub: str, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...) -> int: ... def rindex(self, __sub: str, __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ...) -> int: ... - def rjust(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... - def rpartition(self, __sep: str) -> tuple[str, str, str]: ... - def rsplit(self, sep: str | None = ..., maxsplit: SupportsIndex = ...) -> list[str]: ... - def rstrip(self, __chars: str | None = ...) -> str: ... - def split(self, sep: str | None = ..., maxsplit: SupportsIndex = ...) -> list[str]: ... - def splitlines(self, keepends: bool = ...) -> list[str]: ... + def rjust(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... # type: ignore[misc] + def rpartition(self, __sep: str) -> tuple[str, str, str]: ... # type: ignore[misc] + def rsplit(self, sep: str | None = ..., maxsplit: SupportsIndex = ...) -> list[str]: ... # type: ignore[misc] + def rstrip(self, __chars: str | None = ...) -> str: ... # type: ignore[misc] + def split(self, sep: str | None = ..., maxsplit: SupportsIndex = ...) -> list[str]: ... # type: ignore[misc] + def splitlines(self, keepends: bool = ...) -> list[str]: ... # type: ignore[misc] def startswith( self, __prefix: str | tuple[str, ...], __start: SupportsIndex | None = ..., __end: SupportsIndex | None = ... ) -> bool: ... - def strip(self, __chars: str | None = ...) -> str: ... - def swapcase(self) -> str: ... - def title(self) -> str: ... - def translate(self, __table: Mapping[int, int | str | None] | Sequence[int | str | None]) -> str: ... - def upper(self) -> str: ... - def zfill(self, __width: SupportsIndex) -> str: ... + def strip(self, __chars: str | None = ...) -> str: ... # type: ignore[misc] + def swapcase(self) -> str: ... # type: ignore[misc] + def title(self) -> str: ... # type: ignore[misc] + def translate(self, __table: _TranslateTable) -> str: ... + def upper(self) -> str: ... # type: ignore[misc] + def zfill(self, __width: SupportsIndex) -> str: ... # type: ignore[misc] @staticmethod @overload def maketrans(__x: dict[int, _T] | dict[str, _T] | dict[str | int, _T]) -> dict[int, _T]: ... @staticmethod @overload def maketrans(__x: str, __y: str, __z: str | None = ...) -> dict[int, int | None]: ... - def __add__(self, __s: str) -> str: ... + def __add__(self, __s: str) -> str: ... # type: ignore[misc] # Incompatible with Sequence.__contains__ def __contains__(self, __o: str) -> bool: ... # type: ignore[override] def __eq__(self, __x: object) -> bool: ... def __ge__(self, __x: str) -> bool: ... def __getitem__(self, __i: SupportsIndex | slice) -> str: ... def __gt__(self, __x: str) -> bool: ... - def __hash__(self) -> int: ... - def __iter__(self) -> Iterator[str]: ... + def __iter__(self) -> Iterator[str]: ... # type: ignore[misc] def __le__(self, __x: str) -> bool: ... def __len__(self) -> int: ... def __lt__(self, __x: str) -> bool: ... - def __mod__(self, __x: Any) -> str: ... - def __mul__(self, __n: SupportsIndex) -> str: ... + def __mod__(self, __x: Any) -> str: ... # type: ignore[misc] + def __mul__(self, __n: SupportsIndex) -> str: ... # type: ignore[misc] def __ne__(self, __x: object) -> bool: ... - def __rmul__(self, __n: SupportsIndex) -> str: ... + def __rmul__(self, __n: SupportsIndex) -> str: ... # type: ignore[misc] def __getnewargs__(self) -> tuple[str]: ... class bytes(ByteString): @@ -967,9 +967,9 @@ class dict(MutableMapping[_KT, _VT], Generic[_KT, _VT]): @overload def pop(self, __key: _KT, __default: _VT | _T) -> _VT | _T: ... def __len__(self) -> int: ... - def __getitem__(self, __k: _KT) -> _VT: ... - def __setitem__(self, __k: _KT, __v: _VT) -> None: ... - def __delitem__(self, __v: _KT) -> None: ... + def __getitem__(self, __key: _KT) -> _VT: ... + def __setitem__(self, __key: _KT, __value: _VT) -> None: ... + def __delitem__(self, __key: _KT) -> None: ... def __iter__(self) -> Iterator[_KT]: ... if sys.version_info >= (3, 8): def __reversed__(self) -> Iterator[_KT]: ... @@ -1190,7 +1190,7 @@ else: __locals: Mapping[str, object] | None = ..., ) -> None: ... -def exit(code: object = ...) -> NoReturn: ... +def exit(code: sys._ExitCode = ...) -> NoReturn: ... class filter(Iterator[_T], Generic[_T]): @overload @@ -1457,8 +1457,8 @@ _SupportsSomeKindOfPow = ( # noqa: Y026 # TODO: Use TypeAlias once mypy bugs a ) if sys.version_info >= (3, 8): - @overload - def pow(base: int, exp: int, mod: Literal[0]) -> NoReturn: ... + # TODO: `pow(int, int, Literal[0])` fails at runtime, + # but adding a `NoReturn` overload isn't a good solution for expressing that (see #8566). @overload def pow(base: int, exp: int, mod: int) -> int: ... @overload @@ -1496,8 +1496,6 @@ if sys.version_info >= (3, 8): def pow(base: _SupportsSomeKindOfPow, exp: complex, mod: None = ...) -> complex: ... else: - @overload - def pow(__base: int, __exp: int, __mod: Literal[0]) -> NoReturn: ... @overload def pow(__base: int, __exp: int, __mod: int) -> int: ... @overload @@ -1529,7 +1527,7 @@ else: @overload def pow(__base: _SupportsSomeKindOfPow, __exp: complex, __mod: None = ...) -> complex: ... -def quit(code: object = ...) -> NoReturn: ... +def quit(code: sys._ExitCode = ...) -> NoReturn: ... class reversed(Iterator[_T], Generic[_T]): @overload @@ -1571,11 +1569,11 @@ _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWit # Instead, we special-case the most common examples of this: bool and literal integers. if sys.version_info >= (3, 8): @overload - def sum(__iterable: Iterable[bool | _LiteralInteger], start: int = ...) -> int: ... # type: ignore[misc] + def sum(__iterable: Iterable[bool], start: int = ...) -> int: ... # type: ignore[misc] else: @overload - def sum(__iterable: Iterable[bool | _LiteralInteger], __start: int = ...) -> int: ... # type: ignore[misc] + def sum(__iterable: Iterable[bool], __start: int = ...) -> int: ... # type: ignore[misc] @overload def sum(__iterable: Iterable[_SupportsSumNoDefaultT]) -> _SupportsSumNoDefaultT | Literal[0]: ... @@ -1707,15 +1705,13 @@ class GeneratorExit(BaseException): ... class KeyboardInterrupt(BaseException): ... class SystemExit(BaseException): - code: int + code: sys._ExitCode class Exception(BaseException): ... class StopIteration(Exception): value: Any -_StandardError = Exception - class OSError(Exception): errno: int strerror: str @@ -1730,37 +1726,38 @@ IOError = OSError if sys.platform == "win32": WindowsError = OSError -class ArithmeticError(_StandardError): ... -class AssertionError(_StandardError): ... +class ArithmeticError(Exception): ... +class AssertionError(Exception): ... -class AttributeError(_StandardError): +class AttributeError(Exception): if sys.version_info >= (3, 10): + def __init__(self, *args: object, name: str | None = ..., obj: object = ...) -> None: ... name: str obj: object -class BufferError(_StandardError): ... -class EOFError(_StandardError): ... +class BufferError(Exception): ... +class EOFError(Exception): ... -class ImportError(_StandardError): +class ImportError(Exception): def __init__(self, *args: object, name: str | None = ..., path: str | None = ...) -> None: ... name: str | None path: str | None msg: str # undocumented -class LookupError(_StandardError): ... -class MemoryError(_StandardError): ... +class LookupError(Exception): ... +class MemoryError(Exception): ... -class NameError(_StandardError): +class NameError(Exception): if sys.version_info >= (3, 10): name: str -class ReferenceError(_StandardError): ... -class RuntimeError(_StandardError): ... +class ReferenceError(Exception): ... +class RuntimeError(Exception): ... class StopAsyncIteration(Exception): value: Any -class SyntaxError(_StandardError): +class SyntaxError(Exception): msg: str lineno: int | None offset: int | None @@ -1770,9 +1767,9 @@ class SyntaxError(_StandardError): end_lineno: int | None end_offset: int | None -class SystemError(_StandardError): ... -class TypeError(_StandardError): ... -class ValueError(_StandardError): ... +class SystemError(Exception): ... +class TypeError(Exception): ... +class ValueError(Exception): ... class FloatingPointError(ArithmeticError): ... class OverflowError(ArithmeticError): ... class ZeroDivisionError(ArithmeticError): ... diff --git a/mypy/typeshed/stdlib/calendar.pyi b/mypy/typeshed/stdlib/calendar.pyi index 4faee805333b..74b8d39caf79 100644 --- a/mypy/typeshed/stdlib/calendar.pyi +++ b/mypy/typeshed/stdlib/calendar.pyi @@ -2,6 +2,7 @@ import datetime import sys from collections.abc import Iterable, Sequence from time import struct_time +from typing import ClassVar from typing_extensions import Literal, TypeAlias __all__ = [ @@ -88,6 +89,13 @@ def calendar(theyear: int, w: int = ..., l: int = ..., c: int = ..., m: int = .. def prcal(theyear: int, w: int = ..., l: int = ..., c: int = ..., m: int = ...) -> None: ... class HTMLCalendar(Calendar): + cssclasses: ClassVar[list[str]] + cssclass_noday: ClassVar[str] + cssclasses_weekday_head: ClassVar[list[str]] + cssclass_month_head: ClassVar[str] + cssclass_month: ClassVar[str] + cssclass_year: ClassVar[str] + cssclass_year_head: ClassVar[str] def formatday(self, day: int, weekday: int) -> str: ... def formatweek(self, theweek: int) -> str: ... def formatweekday(self, day: int) -> str: ... @@ -96,13 +104,6 @@ class HTMLCalendar(Calendar): def formatmonth(self, theyear: int, themonth: int, withyear: bool = ...) -> str: ... def formatyear(self, theyear: int, width: int = ...) -> str: ... def formatyearpage(self, theyear: int, width: int = ..., css: str | None = ..., encoding: str | None = ...) -> str: ... - cssclasses: list[str] - cssclass_noday: str - cssclasses_weekday_head: list[str] - cssclass_month_head: str - cssclass_month: str - cssclass_year: str - cssclass_year_head: str class different_locale: def __init__(self, locale: _LocaleType) -> None: ... @@ -111,8 +112,6 @@ class different_locale: class LocaleTextCalendar(TextCalendar): def __init__(self, firstweekday: int = ..., locale: _LocaleType | None = ...) -> None: ... - def formatweekday(self, day: int, width: int) -> str: ... - def formatmonthname(self, theyear: int, themonth: int, width: int, withyear: bool = ...) -> str: ... class LocaleHTMLCalendar(HTMLCalendar): def __init__(self, firstweekday: int = ..., locale: _LocaleType | None = ...) -> None: ... diff --git a/mypy/typeshed/stdlib/cgi.pyi b/mypy/typeshed/stdlib/cgi.pyi index 523b44793941..ce9a15415aab 100644 --- a/mypy/typeshed/stdlib/cgi.pyi +++ b/mypy/typeshed/stdlib/cgi.pyi @@ -2,6 +2,7 @@ import sys from _typeshed import Self, SupportsGetItem, SupportsItemAccess from builtins import list as _list, type as _type from collections.abc import Iterable, Iterator, Mapping +from email.message import Message from types import TracebackType from typing import IO, Any, Protocol @@ -72,7 +73,7 @@ class FieldStorage: keep_blank_values: int strict_parsing: int qs_on_post: str | None - headers: Mapping[str, str] + headers: Mapping[str, str] | Message fp: IO[bytes] encoding: str errors: str @@ -93,7 +94,7 @@ class FieldStorage: def __init__( self, fp: IO[Any] | None = ..., - headers: Mapping[str, str] | None = ..., + headers: Mapping[str, str] | Message | None = ..., outerboundary: bytes = ..., environ: SupportsGetItem[str, str] = ..., keep_blank_values: int = ..., diff --git a/mypy/typeshed/stdlib/codeop.pyi b/mypy/typeshed/stdlib/codeop.pyi index 1c00e13fd501..36af1d297548 100644 --- a/mypy/typeshed/stdlib/codeop.pyi +++ b/mypy/typeshed/stdlib/codeop.pyi @@ -6,10 +6,8 @@ def compile_command(source: str, filename: str = ..., symbol: str = ...) -> Code class Compile: flags: int - def __init__(self) -> None: ... def __call__(self, source: str, filename: str, symbol: str) -> CodeType: ... class CommandCompiler: compiler: Compile - def __init__(self) -> None: ... def __call__(self, source: str, filename: str = ..., symbol: str = ...) -> CodeType | None: ... diff --git a/mypy/typeshed/stdlib/collections/__init__.pyi b/mypy/typeshed/stdlib/collections/__init__.pyi index 40cf999dfae1..37505c256d9c 100644 --- a/mypy/typeshed/stdlib/collections/__init__.pyi +++ b/mypy/typeshed/stdlib/collections/__init__.pyi @@ -8,7 +8,19 @@ if sys.version_info >= (3, 9): from types import GenericAlias if sys.version_info >= (3, 10): - from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Reversible, Sequence + from collections.abc import ( + Callable, + ItemsView, + Iterable, + Iterator, + KeysView, + Mapping, + MutableMapping, + MutableSequence, + Reversible, + Sequence, + ValuesView, + ) else: from _collections_abc import * @@ -301,16 +313,30 @@ class Counter(dict[_T, int], Generic[_T]): def __ge__(self, other: Counter[Any]) -> bool: ... def __gt__(self, other: Counter[Any]) -> bool: ... +# The pure-Python implementations of the "views" classes +# These are exposed at runtime in `collections/__init__.py` +class _OrderedDictKeysView(KeysView[_KT_co], Reversible[_KT_co]): + def __reversed__(self) -> Iterator[_KT_co]: ... + +class _OrderedDictItemsView(ItemsView[_KT_co, _VT_co], Reversible[tuple[_KT_co, _VT_co]]): + def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... + +class _OrderedDictValuesView(ValuesView[_VT_co], Reversible[_VT_co]): + def __reversed__(self) -> Iterator[_VT_co]: ... + +# The C implementations of the "views" classes +# (At runtime, these are called `odict_keys`, `odict_items` and `odict_values`, +# but they are not exposed anywhere) @final -class _OrderedDictKeysView(dict_keys[_KT_co, _VT_co], Reversible[_KT_co]): # type: ignore[misc] +class _odict_keys(dict_keys[_KT_co, _VT_co], Reversible[_KT_co]): # type: ignore[misc] def __reversed__(self) -> Iterator[_KT_co]: ... @final -class _OrderedDictItemsView(dict_items[_KT_co, _VT_co], Reversible[tuple[_KT_co, _VT_co]]): # type: ignore[misc] +class _odict_items(dict_items[_KT_co, _VT_co], Reversible[tuple[_KT_co, _VT_co]]): # type: ignore[misc] def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... @final -class _OrderedDictValuesView(dict_values[_KT_co, _VT_co], Reversible[_VT_co], Generic[_KT_co, _VT_co]): # type: ignore[misc] +class _odict_values(dict_values[_KT_co, _VT_co], Reversible[_VT_co], Generic[_KT_co, _VT_co]): # type: ignore[misc] def __reversed__(self) -> Iterator[_VT_co]: ... class OrderedDict(dict[_KT, _VT], Reversible[_KT], Generic[_KT, _VT]): @@ -318,9 +344,9 @@ class OrderedDict(dict[_KT, _VT], Reversible[_KT], Generic[_KT, _VT]): def move_to_end(self, key: _KT, last: bool = ...) -> None: ... def copy(self: Self) -> Self: ... def __reversed__(self) -> Iterator[_KT]: ... - def keys(self) -> _OrderedDictKeysView[_KT, _VT]: ... - def items(self) -> _OrderedDictItemsView[_KT, _VT]: ... - def values(self) -> _OrderedDictValuesView[_KT, _VT]: ... + def keys(self) -> _odict_keys[_KT, _VT]: ... + def items(self) -> _odict_items[_KT, _VT]: ... + def values(self) -> _odict_values[_KT, _VT]: ... # The signature of OrderedDict.fromkeys should be kept in line with `dict.fromkeys`, modulo positional-only differences. # Like dict.fromkeys, its true signature is not expressible in the current type system. # See #3800 & https://github.com/python/typing/issues/548#issuecomment-683336963. diff --git a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi index 3885abf8db91..3db968878498 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi @@ -35,7 +35,6 @@ _T = TypeVar("_T") _P = ParamSpec("_P") class Future(Generic[_T]): - def __init__(self) -> None: ... def cancel(self) -> bool: ... def cancelled(self) -> bool: ... def running(self) -> bool: ... @@ -90,31 +89,20 @@ def wait(fs: Iterable[Future[_T]], timeout: float | None = ..., return_when: str class _Waiter: event: threading.Event finished_futures: list[Future[Any]] - def __init__(self) -> None: ... def add_result(self, future: Future[Any]) -> None: ... def add_exception(self, future: Future[Any]) -> None: ... def add_cancelled(self, future: Future[Any]) -> None: ... class _AsCompletedWaiter(_Waiter): lock: threading.Lock - def __init__(self) -> None: ... - def add_result(self, future: Future[Any]) -> None: ... - def add_exception(self, future: Future[Any]) -> None: ... - def add_cancelled(self, future: Future[Any]) -> None: ... -class _FirstCompletedWaiter(_Waiter): - def add_result(self, future: Future[Any]) -> None: ... - def add_exception(self, future: Future[Any]) -> None: ... - def add_cancelled(self, future: Future[Any]) -> None: ... +class _FirstCompletedWaiter(_Waiter): ... class _AllCompletedWaiter(_Waiter): num_pending_calls: int stop_on_exception: bool lock: threading.Lock def __init__(self, num_pending_calls: int, stop_on_exception: bool) -> None: ... - def add_result(self, future: Future[Any]) -> None: ... - def add_exception(self, future: Future[Any]) -> None: ... - def add_cancelled(self, future: Future[Any]) -> None: ... class _AcquireFutures: futures: Iterable[Future[Any]] diff --git a/mypy/typeshed/stdlib/concurrent/futures/process.pyi b/mypy/typeshed/stdlib/concurrent/futures/process.pyi index 211107cf357d..a98702d095a2 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/process.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/process.pyi @@ -19,7 +19,6 @@ class _ThreadWakeup: _closed: bool _reader: Connection _writer: Connection - def __init__(self) -> None: ... def close(self) -> None: ... def wakeup(self) -> None: ... def clear(self) -> None: ... diff --git a/mypy/typeshed/stdlib/contextlib.pyi b/mypy/typeshed/stdlib/contextlib.pyi index 6a846ad618c3..ca8830439538 100644 --- a/mypy/typeshed/stdlib/contextlib.pyi +++ b/mypy/typeshed/stdlib/contextlib.pyi @@ -137,7 +137,6 @@ class redirect_stderr(_RedirectStream[_T_io]): ... # In reality this is a subclass of `AbstractContextManager`; # see #7961 for why we don't do that in the stub class ExitStack(metaclass=abc.ABCMeta): - def __init__(self) -> None: ... def enter_context(self, cm: AbstractContextManager[_T]) -> _T: ... def push(self, exit: _CM_EF) -> _CM_EF: ... def callback(self, __callback: Callable[_P, _T], *args: _P.args, **kwds: _P.kwargs) -> Callable[_P, _T]: ... @@ -148,13 +147,14 @@ class ExitStack(metaclass=abc.ABCMeta): self, __exc_type: type[BaseException] | None, __exc_value: BaseException | None, __traceback: TracebackType | None ) -> bool: ... -_ExitCoroFunc: TypeAlias = Callable[[type[BaseException] | None, BaseException | None, TracebackType | None], Awaitable[bool]] +_ExitCoroFunc: TypeAlias = Callable[ + [type[BaseException] | None, BaseException | None, TracebackType | None], Awaitable[bool | None] +] _ACM_EF = TypeVar("_ACM_EF", bound=AbstractAsyncContextManager[Any] | _ExitCoroFunc) # In reality this is a subclass of `AbstractAsyncContextManager`; # see #7961 for why we don't do that in the stub class AsyncExitStack(metaclass=abc.ABCMeta): - def __init__(self) -> None: ... def enter_context(self, cm: AbstractContextManager[_T]) -> _T: ... async def enter_async_context(self, cm: AbstractAsyncContextManager[_T]) -> _T: ... def push(self, exit: _CM_EF) -> _CM_EF: ... diff --git a/mypy/typeshed/stdlib/csv.pyi b/mypy/typeshed/stdlib/csv.pyi index e9552c759c16..8802d6b0a5f5 100644 --- a/mypy/typeshed/stdlib/csv.pyi +++ b/mypy/typeshed/stdlib/csv.pyi @@ -60,24 +60,9 @@ __all__ = [ _T = TypeVar("_T") -class excel(Dialect): - delimiter: str - quotechar: str - doublequote: bool - skipinitialspace: bool - lineterminator: str - quoting: _QuotingType - -class excel_tab(excel): - delimiter: str - -class unix_dialect(Dialect): - delimiter: str - quotechar: str - doublequote: bool - skipinitialspace: bool - lineterminator: str - quoting: _QuotingType +class excel(Dialect): ... +class excel_tab(excel): ... +class unix_dialect(Dialect): ... class DictReader(Generic[_T], Iterator[_DictReadMapping[_T | Any, str | Any]]): fieldnames: Sequence[_T] | None @@ -161,6 +146,5 @@ class DictWriter(Generic[_T]): class Sniffer: preferred: list[str] - def __init__(self) -> None: ... def sniff(self, sample: str, delimiters: str | None = ...) -> type[Dialect]: ... def has_header(self, sample: str) -> bool: ... diff --git a/mypy/typeshed/stdlib/ctypes/__init__.pyi b/mypy/typeshed/stdlib/ctypes/__init__.pyi index 48694fc6cf8a..78f4ee4d5ab3 100644 --- a/mypy/typeshed/stdlib/ctypes/__init__.pyi +++ b/mypy/typeshed/stdlib/ctypes/__init__.pyi @@ -1,4 +1,5 @@ import sys +from _ctypes import RTLD_GLOBAL as RTLD_GLOBAL, RTLD_LOCAL as RTLD_LOCAL from _typeshed import ReadableBuffer, Self, WriteableBuffer from abc import abstractmethod from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence @@ -12,8 +13,6 @@ _T = TypeVar("_T") _DLLT = TypeVar("_DLLT", bound=CDLL) _CT = TypeVar("_CT", bound=_CData) -RTLD_GLOBAL: int -RTLD_LOCAL: int DEFAULT_MODE: int class CDLL: @@ -166,13 +165,10 @@ class _Pointer(Generic[_CT], _PointerLike, _CData): @overload def __init__(self, arg: _CT) -> None: ... @overload - def __getitem__(self, __i: int) -> _CT: ... - @overload - def __getitem__(self, __s: slice) -> list[_CT]: ... - @overload - def __setitem__(self, __i: int, __o: _CT) -> None: ... + def __getitem__(self, __i: int) -> Any: ... @overload - def __setitem__(self, __s: slice, __o: Iterable[_CT]) -> None: ... + def __getitem__(self, __s: slice) -> list[Any]: ... + def __setitem__(self, __i: int, __o: Any) -> None: ... def pointer(__arg: _CT) -> _Pointer[_CT]: ... def resize(obj: _CData, size: int) -> None: ... diff --git a/mypy/typeshed/stdlib/dataclasses.pyi b/mypy/typeshed/stdlib/dataclasses.pyi index 04ae771fc064..560147f9e96b 100644 --- a/mypy/typeshed/stdlib/dataclasses.pyi +++ b/mypy/typeshed/stdlib/dataclasses.pyi @@ -4,7 +4,7 @@ import types from builtins import type as Type # alias to avoid name clashes with fields named "type" from collections.abc import Callable, Iterable, Mapping from typing import Any, Generic, Protocol, TypeVar, overload -from typing_extensions import Literal +from typing_extensions import Literal, TypeAlias if sys.version_info >= (3, 9): from types import GenericAlias @@ -217,7 +217,14 @@ def is_dataclass(obj: Any) -> bool: ... class FrozenInstanceError(AttributeError): ... -class InitVar(Generic[_T]): +if sys.version_info >= (3, 9): + _InitVarMeta: TypeAlias = type +else: + class _InitVarMeta(type): + # Not used, instead `InitVar.__class_getitem__` is called. + def __getitem__(self, params: Any) -> InitVar[Any]: ... + +class InitVar(Generic[_T], metaclass=_InitVarMeta): type: Type[_T] def __init__(self, type: Type[_T]) -> None: ... if sys.version_info >= (3, 9): diff --git a/mypy/typeshed/stdlib/datetime.pyi b/mypy/typeshed/stdlib/datetime.pyi index 780ee941baa5..5926ff0a808e 100644 --- a/mypy/typeshed/stdlib/datetime.pyi +++ b/mypy/typeshed/stdlib/datetime.pyi @@ -201,7 +201,6 @@ class timedelta(SupportsAbs[timedelta]): class datetime(date): min: ClassVar[datetime] max: ClassVar[datetime] - resolution: ClassVar[timedelta] def __new__( cls: type[Self], year: int, @@ -249,8 +248,6 @@ class datetime(date): def utcnow(cls: type[Self]) -> Self: ... @classmethod def combine(cls, date: _Date, time: _Time, tzinfo: _TzInfo | None = ...) -> datetime: ... - @classmethod - def fromisoformat(cls: type[Self], __date_string: str) -> Self: ... def timestamp(self) -> float: ... def utctimetuple(self) -> struct_time: ... def date(self) -> _Date: ... @@ -274,7 +271,6 @@ class datetime(date): else: def astimezone(self, tz: _TzInfo | None = ...) -> datetime: ... - def ctime(self) -> str: ... def isoformat(self, sep: str = ..., timespec: str = ...) -> str: ... @classmethod def strptime(cls, __date_string: str, __format: str) -> datetime: ... @@ -298,7 +294,3 @@ class datetime(date): def __sub__(self, __other: datetime) -> timedelta: ... @overload def __sub__(self, __other: timedelta) -> datetime: ... - if sys.version_info >= (3, 9): - def isocalendar(self) -> _IsoCalendarDate: ... - else: - def isocalendar(self) -> tuple[int, int, int]: ... diff --git a/mypy/typeshed/stdlib/doctest.pyi b/mypy/typeshed/stdlib/doctest.pyi index 382d9578ce80..719551eb77de 100644 --- a/mypy/typeshed/stdlib/doctest.pyi +++ b/mypy/typeshed/stdlib/doctest.pyi @@ -199,24 +199,17 @@ class DocTestCase(unittest.TestCase): self, test: DocTest, optionflags: int = ..., - setUp: Callable[[DocTest], object] | None = ..., - tearDown: Callable[[DocTest], object] | None = ..., + setUp: Callable[[DocTest], Any] | None = ..., + tearDown: Callable[[DocTest], Any] | None = ..., checker: OutputChecker | None = ..., ) -> None: ... - def setUp(self) -> None: ... - def tearDown(self) -> None: ... def runTest(self) -> None: ... def format_failure(self, err: str) -> str: ... - def debug(self) -> None: ... - def id(self) -> str: ... def __eq__(self, other: object) -> bool: ... - def shortDescription(self) -> str: ... class SkipDocTestCase(DocTestCase): def __init__(self, module: types.ModuleType) -> None: ... - def setUp(self) -> None: ... def test_skip(self) -> None: ... - def shortDescription(self) -> str: ... class _DocTestSuite(unittest.TestSuite): ... @@ -228,9 +221,7 @@ def DocTestSuite( **options: Any, ) -> _DocTestSuite: ... -class DocFileCase(DocTestCase): - def id(self) -> str: ... - def format_failure(self, err: str) -> str: ... +class DocFileCase(DocTestCase): ... def DocFileTest( path: str, diff --git a/mypy/typeshed/stdlib/email/_header_value_parser.pyi b/mypy/typeshed/stdlib/email/_header_value_parser.pyi index 00d5c9882429..28a851d2f4e7 100644 --- a/mypy/typeshed/stdlib/email/_header_value_parser.pyi +++ b/mypy/typeshed/stdlib/email/_header_value_parser.pyi @@ -42,11 +42,7 @@ class TokenList(list[TokenList | Terminal]): def pprint(self, indent: str = ...) -> None: ... def ppstr(self, indent: str = ...) -> str: ... -class WhiteSpaceTokenList(TokenList): - @property - def value(self) -> str: ... - @property - def comments(self) -> list[str]: ... +class WhiteSpaceTokenList(TokenList): ... class UnstructuredTokenList(TokenList): token_type: str @@ -84,16 +80,12 @@ class QuotedString(TokenList): class BareQuotedString(QuotedString): token_type: str - @property - def value(self) -> str: ... class Comment(WhiteSpaceTokenList): token_type: str def quote(self, value: Any) -> str: ... @property def content(self) -> str: ... - @property - def comments(self) -> list[str]: ... class AddressList(TokenList): token_type: str @@ -217,8 +209,6 @@ class AddrSpec(TokenList): @property def domain(self) -> str: ... @property - def value(self) -> str: ... - @property def addr_spec(self) -> str: ... class ObsLocalPart(TokenList): @@ -227,18 +217,13 @@ class ObsLocalPart(TokenList): class DisplayName(Phrase): token_type: str - ew_combine_allowed: bool @property def display_name(self) -> str: ... - @property - def value(self) -> str: ... class LocalPart(TokenList): token_type: str as_ew_allowed: bool @property - def value(self) -> str: ... - @property def local_part(self) -> str: ... class DomainLiteral(TokenList): @@ -352,10 +337,7 @@ class ValueTerminal(Terminal): def value(self) -> ValueTerminal: ... def startswith_fws(self) -> bool: ... -class EWWhiteSpaceTerminal(WhiteSpaceTerminal): - @property - def value(self) -> str: ... - +class EWWhiteSpaceTerminal(WhiteSpaceTerminal): ... class _InvalidEwError(HeaderParseError): ... DOT: Final[ValueTerminal] diff --git a/mypy/typeshed/stdlib/email/contentmanager.pyi b/mypy/typeshed/stdlib/email/contentmanager.pyi index 3ac665eaa7bf..3214f1a4781d 100644 --- a/mypy/typeshed/stdlib/email/contentmanager.pyi +++ b/mypy/typeshed/stdlib/email/contentmanager.pyi @@ -3,7 +3,6 @@ from email.message import Message from typing import Any class ContentManager: - def __init__(self) -> None: ... def get_content(self, msg: Message, *args: Any, **kw: Any) -> Any: ... def set_content(self, msg: Message, obj: Any, *args: Any, **kw: Any) -> Any: ... def add_get_handler(self, key: str, handler: Callable[..., Any]) -> None: ... diff --git a/mypy/typeshed/stdlib/email/parser.pyi b/mypy/typeshed/stdlib/email/parser.pyi index dcd346c1b46d..bf51c45728fd 100644 --- a/mypy/typeshed/stdlib/email/parser.pyi +++ b/mypy/typeshed/stdlib/email/parser.pyi @@ -11,17 +11,11 @@ class Parser: def parse(self, fp: TextIO, headersonly: bool = ...) -> Message: ... def parsestr(self, text: str, headersonly: bool = ...) -> Message: ... -class HeaderParser(Parser): - def __init__(self, _class: Callable[[], Message] | None = ..., *, policy: Policy = ...) -> None: ... - def parse(self, fp: TextIO, headersonly: bool = ...) -> Message: ... - def parsestr(self, text: str, headersonly: bool = ...) -> Message: ... - -class BytesHeaderParser(BytesParser): - def __init__(self, _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> None: ... - def parse(self, fp: BinaryIO, headersonly: bool = ...) -> Message: ... - def parsebytes(self, text: bytes, headersonly: bool = ...) -> Message: ... +class HeaderParser(Parser): ... class BytesParser: def __init__(self, _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> None: ... def parse(self, fp: BinaryIO, headersonly: bool = ...) -> Message: ... def parsebytes(self, text: bytes, headersonly: bool = ...) -> Message: ... + +class BytesHeaderParser(BytesParser): ... diff --git a/mypy/typeshed/stdlib/encodings/utf_8_sig.pyi b/mypy/typeshed/stdlib/encodings/utf_8_sig.pyi index bf52e8a6f3d3..ad0d5bdc4fc7 100644 --- a/mypy/typeshed/stdlib/encodings/utf_8_sig.pyi +++ b/mypy/typeshed/stdlib/encodings/utf_8_sig.pyi @@ -3,23 +3,17 @@ import codecs class IncrementalEncoder(codecs.IncrementalEncoder): def __init__(self, errors: str = ...) -> None: ... def encode(self, input: str, final: bool = ...) -> bytes: ... - def reset(self) -> None: ... def getstate(self) -> int: ... # type: ignore[override] def setstate(self, state: int) -> None: ... # type: ignore[override] class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def __init__(self, errors: str = ...) -> None: ... def _buffer_decode(self, input: bytes, errors: str | None, final: bool) -> tuple[str, int]: ... - def reset(self) -> None: ... - def getstate(self) -> tuple[bytes, int]: ... - def setstate(self, state: tuple[bytes, int]) -> None: ... class StreamWriter(codecs.StreamWriter): - def reset(self) -> None: ... def encode(self, input: str, errors: str | None = ...) -> tuple[bytes, int]: ... class StreamReader(codecs.StreamReader): - def reset(self) -> None: ... def decode(self, input: bytes, errors: str | None = ...) -> tuple[str, int]: ... def getregentry() -> codecs.CodecInfo: ... diff --git a/mypy/typeshed/stdlib/formatter.pyi b/mypy/typeshed/stdlib/formatter.pyi index 0aac0a5f918c..388dbd6071ac 100644 --- a/mypy/typeshed/stdlib/formatter.pyi +++ b/mypy/typeshed/stdlib/formatter.pyi @@ -64,7 +64,6 @@ class AbstractFormatter: def assert_line_data(self, flag: int = ...) -> None: ... class NullWriter: - def __init__(self) -> None: ... def flush(self) -> None: ... def new_alignment(self, align: str | None) -> None: ... def new_font(self, font: _FontType) -> None: ... @@ -78,28 +77,12 @@ class NullWriter: def send_flowing_data(self, data: str) -> None: ... def send_literal_data(self, data: str) -> None: ... -class AbstractWriter(NullWriter): - def new_alignment(self, align: str | None) -> None: ... - def new_font(self, font: _FontType) -> None: ... - def new_margin(self, margin: int, level: int) -> None: ... - def new_spacing(self, spacing: str | None) -> None: ... - def new_styles(self, styles: tuple[Any, ...]) -> None: ... - def send_paragraph(self, blankline: int) -> None: ... - def send_line_break(self) -> None: ... - def send_hor_rule(self, *args: Any, **kw: Any) -> None: ... - def send_label_data(self, data: str) -> None: ... - def send_flowing_data(self, data: str) -> None: ... - def send_literal_data(self, data: str) -> None: ... +class AbstractWriter(NullWriter): ... class DumbWriter(NullWriter): file: IO[str] maxcol: int def __init__(self, file: IO[str] | None = ..., maxcol: int = ...) -> None: ... def reset(self) -> None: ... - def send_paragraph(self, blankline: int) -> None: ... - def send_line_break(self) -> None: ... - def send_hor_rule(self, *args: Any, **kw: Any) -> None: ... - def send_literal_data(self, data: str) -> None: ... - def send_flowing_data(self, data: str) -> None: ... def test(file: str | None = ...) -> None: ... diff --git a/mypy/typeshed/stdlib/gzip.pyi b/mypy/typeshed/stdlib/gzip.pyi index abf12925aea2..75a70a5e7a07 100644 --- a/mypy/typeshed/stdlib/gzip.pyi +++ b/mypy/typeshed/stdlib/gzip.pyi @@ -15,8 +15,14 @@ _ReadBinaryMode: TypeAlias = Literal["r", "rb"] _WriteBinaryMode: TypeAlias = Literal["a", "ab", "w", "wb", "x", "xb"] _OpenTextMode: TypeAlias = Literal["rt", "at", "wt", "xt"] -READ: Literal[1] -WRITE: Literal[2] +READ: Literal[1] # undocumented +WRITE: Literal[2] # undocumented + +FTEXT: int # actually Literal[1] # undocumented +FHCRC: int # actually Literal[2] # undocumented +FEXTRA: int # actually Literal[4] # undocumented +FNAME: int # actually Literal[8] # undocumented +FCOMMENT: int # actually Literal[16] # undocumented class _ReadableFileobj(Protocol): def read(self, __n: int) -> bytes: ... @@ -142,21 +148,15 @@ class GzipFile(_compression.BaseStream): def read(self, size: int | None = ...) -> bytes: ... def read1(self, size: int = ...) -> bytes: ... def peek(self, n: int) -> bytes: ... - @property - def closed(self) -> bool: ... def close(self) -> None: ... def flush(self, zlib_mode: int = ...) -> None: ... def fileno(self) -> int: ... def rewind(self) -> None: ... - def readable(self) -> bool: ... - def writable(self) -> bool: ... - def seekable(self) -> bool: ... def seek(self, offset: int, whence: int = ...) -> int: ... def readline(self, size: int | None = ...) -> bytes: ... class _GzipReader(_compression.DecompressReader): def __init__(self, fp: _ReadableFileobj) -> None: ... - def read(self, size: int = ...) -> bytes: ... if sys.version_info >= (3, 8): def compress(data: bytes, compresslevel: int = ..., *, mtime: float | None = ...) -> bytes: ... diff --git a/mypy/typeshed/stdlib/html/parser.pyi b/mypy/typeshed/stdlib/html/parser.pyi index 2948eadc9800..6dde9f705978 100644 --- a/mypy/typeshed/stdlib/html/parser.pyi +++ b/mypy/typeshed/stdlib/html/parser.pyi @@ -7,8 +7,6 @@ class HTMLParser(ParserBase): def __init__(self, *, convert_charrefs: bool = ...) -> None: ... def feed(self, data: str) -> None: ... def close(self) -> None: ... - def reset(self) -> None: ... - def getpos(self) -> tuple[int, int]: ... def get_starttag_text(self) -> str | None: ... def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: ... def handle_endtag(self, tag: str) -> None: ... @@ -19,7 +17,6 @@ class HTMLParser(ParserBase): def handle_comment(self, data: str) -> None: ... def handle_decl(self, decl: str) -> None: ... def handle_pi(self, data: str) -> None: ... - def unknown_decl(self, data: str) -> None: ... CDATA_CONTENT_ELEMENTS: tuple[str, ...] def check_for_whole_start_tag(self, i: int) -> int: ... # undocumented def clear_cdata_mode(self) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/http/client.pyi b/mypy/typeshed/stdlib/http/client.pyi index 08c3f2c8be0b..2ce52eac9ad9 100644 --- a/mypy/typeshed/stdlib/http/client.pyi +++ b/mypy/typeshed/stdlib/http/client.pyi @@ -125,7 +125,6 @@ class HTTPResponse(io.BufferedIOBase, BinaryIO): @overload def getheader(self, name: str, default: _T) -> str | _T: ... def getheaders(self) -> list[tuple[str, str]]: ... - def fileno(self) -> int: ... def isclosed(self) -> bool: ... def __iter__(self) -> Iterator[bytes]: ... def __enter__(self: Self) -> Self: ... diff --git a/mypy/typeshed/stdlib/http/server.pyi b/mypy/typeshed/stdlib/http/server.pyi index e73497bb18bc..40c94bf62f30 100644 --- a/mypy/typeshed/stdlib/http/server.pyi +++ b/mypy/typeshed/stdlib/http/server.pyi @@ -11,12 +11,10 @@ class HTTPServer(socketserver.TCPServer): server_name: str server_port: int -class ThreadingHTTPServer(socketserver.ThreadingMixIn, HTTPServer): - daemon_threads: bool # undocumented +class ThreadingHTTPServer(socketserver.ThreadingMixIn, HTTPServer): ... class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): client_address: tuple[str, int] - server: socketserver.BaseServer close_connection: bool requestline: str command: str @@ -34,7 +32,6 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): weekdayname: ClassVar[Sequence[str]] # undocumented monthname: ClassVar[Sequence[str | None]] # undocumented def __init__(self, request: bytes, client_address: tuple[str, int], server: socketserver.BaseServer) -> None: ... - def handle(self) -> None: ... def handle_one_request(self) -> None: ... def handle_expect_100(self) -> bool: ... def send_error(self, code: int, message: str | None = ..., explain: str | None = ...) -> None: ... @@ -53,7 +50,6 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): def parse_request(self) -> bool: ... # undocumented class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): - server_version: str extensions_map: dict[str, str] def __init__( self, request: bytes, client_address: tuple[str, int], server: socketserver.BaseServer, directory: str | None = ... diff --git a/mypy/typeshed/stdlib/imaplib.pyi b/mypy/typeshed/stdlib/imaplib.pyi index a313b20a999f..bd3d0777db15 100644 --- a/mypy/typeshed/stdlib/imaplib.pyi +++ b/mypy/typeshed/stdlib/imaplib.pyi @@ -4,6 +4,7 @@ import time from _typeshed import Self from builtins import list as _list # conflicts with a method named "list" from collections.abc import Callable +from datetime import datetime from re import Pattern from socket import socket as _socket from ssl import SSLContext, SSLSocket @@ -128,9 +129,6 @@ class IMAP4_SSL(IMAP4): certfile: str | None = ..., ssl_context: SSLContext | None = ..., ) -> None: ... - host: str - port: int - sock: _socket sslobj: SSLSocket file: IO[Any] if sys.version_info >= (3, 9): @@ -138,19 +136,11 @@ class IMAP4_SSL(IMAP4): else: def open(self, host: str = ..., port: int | None = ...) -> None: ... - def read(self, size: int) -> bytes: ... - def readline(self) -> bytes: ... - def send(self, data: bytes) -> None: ... - def shutdown(self) -> None: ... - def socket(self) -> _socket: ... def ssl(self) -> SSLSocket: ... class IMAP4_stream(IMAP4): command: str def __init__(self, command: str) -> None: ... - host: str - port: int - sock: _socket file: IO[Any] process: subprocess.Popen[bytes] writefile: IO[Any] @@ -160,11 +150,6 @@ class IMAP4_stream(IMAP4): else: def open(self, host: str | None = ..., port: int | None = ...) -> None: ... - def read(self, size: int) -> bytes: ... - def readline(self) -> bytes: ... - def send(self, data: bytes) -> None: ... - def shutdown(self) -> None: ... - class _Authenticator: mech: Callable[[bytes], bytes] def __init__(self, mechinst: Callable[[bytes], bytes]) -> None: ... @@ -175,4 +160,4 @@ class _Authenticator: def Internaldate2tuple(resp: bytes) -> time.struct_time: ... def Int2AP(num: int) -> str: ... def ParseFlags(resp: bytes) -> tuple[bytes, ...]: ... -def Time2Internaldate(date_time: float | time.struct_time | str) -> str: ... +def Time2Internaldate(date_time: float | time.struct_time | time._TimeTuple | datetime | str) -> str: ... diff --git a/mypy/typeshed/stdlib/importlib/abc.pyi b/mypy/typeshed/stdlib/importlib/abc.pyi index b46d42a4199a..708037305c67 100644 --- a/mypy/typeshed/stdlib/importlib/abc.pyi +++ b/mypy/typeshed/stdlib/importlib/abc.pyi @@ -1,14 +1,6 @@ import sys import types -from _typeshed import ( - OpenBinaryMode, - OpenBinaryModeReading, - OpenBinaryModeUpdating, - OpenBinaryModeWriting, - OpenTextMode, - StrOrBytesPath, - StrPath, -) +from _typeshed import OpenBinaryMode, OpenBinaryModeReading, OpenBinaryModeUpdating, OpenBinaryModeWriting, OpenTextMode from abc import ABCMeta, abstractmethod from collections.abc import Iterator, Mapping, Sequence from importlib.machinery import ModuleSpec @@ -36,6 +28,14 @@ _Path: TypeAlias = bytes | str class Finder(metaclass=ABCMeta): ... +class Loader(metaclass=ABCMeta): + def load_module(self, fullname: str) -> types.ModuleType: ... + def module_repr(self, module: types.ModuleType) -> str: ... + def create_module(self, spec: ModuleSpec) -> types.ModuleType | None: ... + # Not defined on the actual class for backwards-compatibility reasons, + # but expected in new code. + def exec_module(self, module: types.ModuleType) -> None: ... + class ResourceLoader(Loader): @abstractmethod def get_data(self, path: _Path) -> bytes: ... @@ -43,7 +43,6 @@ class ResourceLoader(Loader): class InspectLoader(Loader): def is_package(self, fullname: str) -> bool: ... def get_code(self, fullname: str) -> types.CodeType | None: ... - def load_module(self, fullname: str) -> types.ModuleType: ... @abstractmethod def get_source(self, fullname: str) -> str | None: ... def exec_module(self, module: types.ModuleType) -> None: ... @@ -53,7 +52,6 @@ class InspectLoader(Loader): class ExecutionLoader(InspectLoader): @abstractmethod def get_filename(self, fullname: str) -> _Path: ... - def get_code(self, fullname: str) -> types.CodeType | None: ... class SourceLoader(ResourceLoader, ExecutionLoader, metaclass=ABCMeta): def path_mtime(self, path: _Path) -> float: ... @@ -77,14 +75,6 @@ class PathEntryFinder(Finder): # Not defined on the actual class, but expected to exist. def find_spec(self, fullname: str, target: types.ModuleType | None = ...) -> ModuleSpec | None: ... -class Loader(metaclass=ABCMeta): - def load_module(self, fullname: str) -> types.ModuleType: ... - def module_repr(self, module: types.ModuleType) -> str: ... - def create_module(self, spec: ModuleSpec) -> types.ModuleType | None: ... - # Not defined on the actual class for backwards-compatibility reasons, - # but expected in new code. - def exec_module(self, module: types.ModuleType) -> None: ... - class FileLoader(ResourceLoader, ExecutionLoader, metaclass=ABCMeta): name: str path: _Path @@ -95,9 +85,9 @@ class FileLoader(ResourceLoader, ExecutionLoader, metaclass=ABCMeta): class ResourceReader(metaclass=ABCMeta): @abstractmethod - def open_resource(self, resource: StrOrBytesPath) -> IO[bytes]: ... + def open_resource(self, resource: str) -> IO[bytes]: ... @abstractmethod - def resource_path(self, resource: StrOrBytesPath) -> str: ... + def resource_path(self, resource: str) -> str: ... if sys.version_info >= (3, 10): @abstractmethod def is_resource(self, path: str) -> bool: ... @@ -117,8 +107,12 @@ if sys.version_info >= (3, 9): def is_file(self) -> bool: ... @abstractmethod def iterdir(self) -> Iterator[Traversable]: ... - @abstractmethod - def joinpath(self, child: StrPath) -> Traversable: ... + if sys.version_info >= (3, 11): + @abstractmethod + def joinpath(self, *descendants: str) -> Traversable: ... + else: + @abstractmethod + def joinpath(self, child: str) -> Traversable: ... # The .open method comes from pathlib.pyi and should be kept in sync. @overload @abstractmethod @@ -182,7 +176,7 @@ if sys.version_info >= (3, 9): @property def name(self) -> str: ... @abstractmethod - def __truediv__(self, child: StrPath) -> Traversable: ... + def __truediv__(self, child: str) -> Traversable: ... @abstractmethod def read_bytes(self) -> bytes: ... @abstractmethod @@ -191,7 +185,7 @@ if sys.version_info >= (3, 9): class TraversableResources(ResourceReader): @abstractmethod def files(self) -> Traversable: ... - def open_resource(self, resource: StrPath) -> BufferedReader: ... # type: ignore[override] + def open_resource(self, resource: str) -> BufferedReader: ... # type: ignore[override] def resource_path(self, resource: Any) -> NoReturn: ... - def is_resource(self, path: StrPath) -> bool: ... + def is_resource(self, path: str) -> bool: ... def contents(self) -> Iterator[str]: ... diff --git a/mypy/typeshed/stdlib/importlib/machinery.pyi b/mypy/typeshed/stdlib/importlib/machinery.pyi index 09abdc6f34fd..ba6ed30629e0 100644 --- a/mypy/typeshed/stdlib/importlib/machinery.pyi +++ b/mypy/typeshed/stdlib/importlib/machinery.pyi @@ -145,6 +145,5 @@ class ExtensionFileLoader(importlib.abc.ExecutionLoader): def get_source(self, fullname: str) -> None: ... def create_module(self, spec: ModuleSpec) -> types.ModuleType: ... def exec_module(self, module: types.ModuleType) -> None: ... - def is_package(self, fullname: str) -> bool: ... def get_code(self, fullname: str) -> None: ... def __eq__(self, other: object) -> bool: ... diff --git a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi index 99fecb41497d..01e35db5815e 100644 --- a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi @@ -41,6 +41,9 @@ class _EntryPointBase(NamedTuple): class EntryPoint(_EntryPointBase): pattern: ClassVar[Pattern[str]] + if sys.version_info >= (3, 11): + def __init__(self, name: str, value: str, group: str) -> None: ... + def load(self) -> Any: ... # Callable[[], Any] or an importable module @property def extras(self) -> list[str]: ... diff --git a/mypy/typeshed/stdlib/importlib/util.pyi b/mypy/typeshed/stdlib/importlib/util.pyi index dca4778fd416..4d75032ab44a 100644 --- a/mypy/typeshed/stdlib/importlib/util.pyi +++ b/mypy/typeshed/stdlib/importlib/util.pyi @@ -35,7 +35,6 @@ class LazyLoader(importlib.abc.Loader): def __init__(self, loader: importlib.abc.Loader) -> None: ... @classmethod def factory(cls, loader: importlib.abc.Loader) -> Callable[..., LazyLoader]: ... - def create_module(self, spec: importlib.machinery.ModuleSpec) -> types.ModuleType | None: ... def exec_module(self, module: types.ModuleType) -> None: ... def source_hash(source_bytes: bytes) -> int: ... diff --git a/mypy/typeshed/stdlib/inspect.pyi b/mypy/typeshed/stdlib/inspect.pyi index 7f9667c6ebed..b97bc601271a 100644 --- a/mypy/typeshed/stdlib/inspect.pyi +++ b/mypy/typeshed/stdlib/inspect.pyi @@ -401,7 +401,7 @@ class BoundArguments: # seem to be supporting this at the moment: # _ClassTreeItem = list[_ClassTreeItem] | Tuple[type, Tuple[type, ...]] def getclasstree(classes: list[type], unique: bool = ...) -> list[Any]: ... -def walktree(classes: list[type], children: dict[type[Any], list[type]], parent: type[Any] | None) -> list[Any]: ... +def walktree(classes: list[type], children: Mapping[type[Any], list[type]], parent: type[Any] | None) -> list[Any]: ... class Arguments(NamedTuple): args: list[str] @@ -446,8 +446,8 @@ if sys.version_info < (3, 11): varkw: str | None = ..., defaults: tuple[Any, ...] | None = ..., kwonlyargs: Sequence[str] | None = ..., - kwonlydefaults: dict[str, Any] | None = ..., - annotations: dict[str, Any] = ..., + kwonlydefaults: Mapping[str, Any] | None = ..., + annotations: Mapping[str, Any] = ..., formatarg: Callable[[str], str] = ..., formatvarargs: Callable[[str], str] = ..., formatvarkw: Callable[[str], str] = ..., @@ -460,7 +460,7 @@ def formatargvalues( args: list[str], varargs: str | None, varkw: str | None, - locals: dict[str, Any] | None, + locals: Mapping[str, Any] | None, formatarg: Callable[[str], str] | None = ..., formatvarargs: Callable[[str], str] | None = ..., formatvarkw: Callable[[str], str] | None = ..., diff --git a/mypy/typeshed/stdlib/io.pyi b/mypy/typeshed/stdlib/io.pyi index f47a9ddf334c..3e9a6cd6861d 100644 --- a/mypy/typeshed/stdlib/io.pyi +++ b/mypy/typeshed/stdlib/io.pyi @@ -117,7 +117,6 @@ class BufferedReader(BufferedIOBase, BinaryIO): def __enter__(self: Self) -> Self: ... def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... def peek(self, __size: int = ...) -> bytes: ... - def read1(self, __size: int = ...) -> bytes: ... class BufferedWriter(BufferedIOBase, BinaryIO): def __enter__(self: Self) -> Self: ... @@ -126,9 +125,7 @@ class BufferedWriter(BufferedIOBase, BinaryIO): class BufferedRandom(BufferedReader, BufferedWriter): def __enter__(self: Self) -> Self: ... - def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... - def seek(self, __target: int, __whence: int = ...) -> int: ... - def read1(self, __size: int = ...) -> bytes: ... + def seek(self, __target: int, __whence: int = ...) -> int: ... # stubtest needs this class BufferedRWPair(BufferedIOBase): def __init__(self, reader: RawIOBase, writer: RawIOBase, buffer_size: int = ...) -> None: ... @@ -146,7 +143,6 @@ class TextIOBase(IOBase): def readline(self, __size: int = ...) -> str: ... # type: ignore[override] def readlines(self, __hint: int = ...) -> list[str]: ... # type: ignore[override] def read(self, __size: int | None = ...) -> str: ... - def tell(self) -> int: ... class TextIOWrapper(TextIOBase, TextIO): def __init__( @@ -182,7 +178,7 @@ class TextIOWrapper(TextIOBase, TextIO): def writelines(self, __lines: Iterable[str]) -> None: ... # type: ignore[override] def readline(self, __size: int = ...) -> str: ... # type: ignore[override] def readlines(self, __hint: int = ...) -> list[str]: ... # type: ignore[override] - def seek(self, __cookie: int, __whence: int = ...) -> int: ... + def seek(self, __cookie: int, __whence: int = ...) -> int: ... # stubtest needs this class StringIO(TextIOWrapper): def __init__(self, initial_value: str | None = ..., newline: str | None = ...) -> None: ... diff --git a/mypy/typeshed/stdlib/ipaddress.pyi b/mypy/typeshed/stdlib/ipaddress.pyi index d324f52ac25a..2c0292d6fbae 100644 --- a/mypy/typeshed/stdlib/ipaddress.pyi +++ b/mypy/typeshed/stdlib/ipaddress.pyi @@ -15,7 +15,9 @@ _RawIPAddress: TypeAlias = int | str | bytes | IPv4Address | IPv6Address _RawNetworkPart: TypeAlias = IPv4Network | IPv6Network | IPv4Interface | IPv6Interface def ip_address(address: _RawIPAddress) -> IPv4Address | IPv6Address: ... -def ip_network(address: _RawIPAddress | _RawNetworkPart, strict: bool = ...) -> IPv4Network | IPv6Network: ... +def ip_network( + address: _RawIPAddress | _RawNetworkPart | tuple[_RawIPAddress] | tuple[_RawIPAddress, int], strict: bool = ... +) -> IPv4Network | IPv6Network: ... def ip_interface(address: _RawIPAddress | _RawNetworkPart) -> IPv4Interface | IPv6Interface: ... class _IPAddressBase: diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/grammar.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/grammar.pyi index 4d298ec6972c..aa0dd687659d 100644 --- a/mypy/typeshed/stdlib/lib2to3/pgen2/grammar.pyi +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/grammar.pyi @@ -15,7 +15,6 @@ class Grammar: tokens: dict[int, int] symbol2label: dict[str, int] start: int - def __init__(self) -> None: ... def dump(self, filename: StrPath) -> None: ... def load(self, filename: StrPath) -> None: ... def copy(self: Self) -> Self: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/pgen.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/pgen.pyi index e3ea07432d70..84ee7ae98bd0 100644 --- a/mypy/typeshed/stdlib/lib2to3/pgen2/pgen.pyi +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/pgen.pyi @@ -32,7 +32,6 @@ class ParserGenerator: class NFAState: arcs: list[tuple[str | None, NFAState]] - def __init__(self) -> None: ... def addarc(self, next: NFAState, label: str | None = ...) -> None: ... class DFAState: diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/tokenize.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/tokenize.pyi index c9ad1e7bb411..2a9c3fbba821 100644 --- a/mypy/typeshed/stdlib/lib2to3/pgen2/tokenize.pyi +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/tokenize.pyi @@ -87,7 +87,6 @@ class Untokenizer: tokens: list[str] prev_row: int prev_col: int - def __init__(self) -> None: ... def add_whitespace(self, start: _Coord) -> None: ... def untokenize(self, iterable: Iterable[_TokenInfo]) -> str: ... def compat(self, token: tuple[int, str], iterable: Iterable[_TokenInfo]) -> None: ... diff --git a/mypy/typeshed/stdlib/logging/__init__.pyi b/mypy/typeshed/stdlib/logging/__init__.pyi index 0d3e80ddcf00..575fd8f9ee4b 100644 --- a/mypy/typeshed/stdlib/logging/__init__.pyi +++ b/mypy/typeshed/stdlib/logging/__init__.pyi @@ -81,7 +81,6 @@ _nameToLevel: dict[str, int] class Filterer: filters: list[Filter] - def __init__(self) -> None: ... def addFilter(self, filter: _FilterType) -> None: ... def removeFilter(self, filter: _FilterType) -> None: ... def filter(self, record: LogRecord) -> bool: ... @@ -272,7 +271,6 @@ class Logger(Filterer): stack_info: bool = ..., ) -> None: ... # undocumented fatal = critical - def filter(self, record: LogRecord) -> bool: ... def addHandler(self, hdlr: Handler) -> None: ... def removeHandler(self, hdlr: Handler) -> None: ... if sys.version_info >= (3, 8): @@ -319,7 +317,6 @@ class Handler(Filterer): def release(self) -> None: ... def setLevel(self, level: _Level) -> None: ... def setFormatter(self, fmt: Formatter | None) -> None: ... - def filter(self, record: LogRecord) -> bool: ... def flush(self) -> None: ... def close(self) -> None: ... def handle(self, record: LogRecord) -> bool: ... diff --git a/mypy/typeshed/stdlib/lzma.pyi b/mypy/typeshed/stdlib/lzma.pyi index d4c7977b8d0a..868da0f05567 100644 --- a/mypy/typeshed/stdlib/lzma.pyi +++ b/mypy/typeshed/stdlib/lzma.pyi @@ -116,20 +116,12 @@ class LZMAFile(io.BufferedIOBase, IO[bytes]): filters: _FilterChain | None = ..., ) -> None: ... def __enter__(self: Self) -> Self: ... - def close(self) -> None: ... - @property - def closed(self) -> bool: ... - def fileno(self) -> int: ... - def seekable(self) -> bool: ... - def readable(self) -> bool: ... - def writable(self) -> bool: ... def peek(self, size: int = ...) -> bytes: ... def read(self, size: int | None = ...) -> bytes: ... def read1(self, size: int = ...) -> bytes: ... def readline(self, size: int | None = ...) -> bytes: ... def write(self, data: ReadableBuffer) -> int: ... def seek(self, offset: int, whence: int = ...) -> int: ... - def tell(self) -> int: ... @overload def open( diff --git a/mypy/typeshed/stdlib/multiprocessing/connection.pyi b/mypy/typeshed/stdlib/multiprocessing/connection.pyi index 489e8bd9a9f1..cc9f5cf8f890 100644 --- a/mypy/typeshed/stdlib/multiprocessing/connection.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/connection.pyi @@ -58,4 +58,12 @@ def wait( object_list: Iterable[Connection | socket.socket | int], timeout: float | None = ... ) -> list[Connection | socket.socket | int]: ... def Client(address: _Address, family: str | None = ..., authkey: bytes | None = ...) -> Connection: ... -def Pipe(duplex: bool = ...) -> tuple[_ConnectionBase, _ConnectionBase]: ... + +# N.B. Keep this in sync with multiprocessing.context.BaseContext.Pipe. +# _ConnectionBase is the common base class of Connection and PipeConnection +# and can be used in cross-platform code. +if sys.platform != "win32": + def Pipe(duplex: bool = ...) -> tuple[Connection, Connection]: ... + +else: + def Pipe(duplex: bool = ...) -> tuple[PipeConnection, PipeConnection]: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/context.pyi b/mypy/typeshed/stdlib/multiprocessing/context.pyi index 16b7cfe9e890..f6380e2cfcbf 100644 --- a/mypy/typeshed/stdlib/multiprocessing/context.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/context.pyi @@ -4,7 +4,6 @@ from collections.abc import Callable, Iterable, Sequence from ctypes import _CData from logging import Logger from multiprocessing import popen_fork, popen_forkserver, popen_spawn_posix, popen_spawn_win32, queues, synchronize -from multiprocessing.connection import _ConnectionBase from multiprocessing.managers import SyncManager from multiprocessing.pool import Pool as _Pool from multiprocessing.process import BaseProcess @@ -12,6 +11,11 @@ from multiprocessing.sharedctypes import SynchronizedArray, SynchronizedBase from typing import Any, ClassVar, TypeVar, overload from typing_extensions import Literal, TypeAlias +if sys.platform != "win32": + from multiprocessing.connection import Connection +else: + from multiprocessing.connection import PipeConnection + if sys.version_info >= (3, 8): __all__ = () else: @@ -43,7 +47,15 @@ class BaseContext: def active_children() -> list[BaseProcess]: ... def cpu_count(self) -> int: ... def Manager(self) -> SyncManager: ... - def Pipe(self, duplex: bool = ...) -> tuple[_ConnectionBase, _ConnectionBase]: ... + + # N.B. Keep this in sync with multiprocessing.connection.Pipe. + # _ConnectionBase is the common base class of Connection and PipeConnection + # and can be used in cross-platform code. + if sys.platform != "win32": + def Pipe(self, duplex: bool = ...) -> tuple[Connection, Connection]: ... + else: + def Pipe(self, duplex: bool = ...) -> tuple[PipeConnection, PipeConnection]: ... + def Barrier( self, parties: int, action: Callable[..., object] | None = ..., timeout: float | None = ... ) -> synchronize.Barrier: ... @@ -137,7 +149,6 @@ class Process(BaseProcess): class DefaultContext(BaseContext): Process: ClassVar[type[Process]] def __init__(self, context: BaseContext) -> None: ... - def set_start_method(self, method: str | None, force: bool = ...) -> None: ... def get_start_method(self, allow_none: bool = ...) -> str: ... def get_all_start_methods(self) -> list[str]: ... if sys.version_info < (3, 8): diff --git a/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi b/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi index 93777d926ca2..10269dfbba29 100644 --- a/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi @@ -9,7 +9,6 @@ MAXFDS_TO_SEND: int SIGNED_STRUCT: Struct class ForkServer: - def __init__(self) -> None: ... def set_forkserver_preload(self, modules_names: list[str]) -> None: ... def get_inherited_fds(self) -> list[int] | None: ... def connect_to_new_process(self, fds: Sequence[int]) -> tuple[int, int]: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/managers.pyi b/mypy/typeshed/stdlib/multiprocessing/managers.pyi index d953785d81cb..2630e5864520 100644 --- a/mypy/typeshed/stdlib/multiprocessing/managers.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/managers.pyi @@ -68,9 +68,9 @@ class ValueProxy(BaseProxy, Generic[_T]): class DictProxy(BaseProxy, MutableMapping[_KT, _VT]): __builtins__: ClassVar[dict[str, Any]] def __len__(self) -> int: ... - def __getitem__(self, __k: _KT) -> _VT: ... - def __setitem__(self, __k: _KT, __v: _VT) -> None: ... - def __delitem__(self, __v: _KT) -> None: ... + def __getitem__(self, __key: _KT) -> _VT: ... + def __setitem__(self, __key: _KT, __value: _VT) -> None: ... + def __delitem__(self, __key: _KT) -> None: ... def __iter__(self) -> Iterator[_KT]: ... def copy(self) -> dict[_KT, _VT]: ... @overload @@ -82,8 +82,8 @@ class DictProxy(BaseProxy, MutableMapping[_KT, _VT]): @overload def pop(self, __key: _KT, __default: _VT | _T) -> _VT | _T: ... def keys(self) -> list[_KT]: ... # type: ignore[override] - def values(self) -> list[tuple[_KT, _VT]]: ... # type: ignore[override] - def items(self) -> list[_VT]: ... # type: ignore[override] + def items(self) -> list[tuple[_KT, _VT]]: ... # type: ignore[override] + def values(self) -> list[_VT]: ... # type: ignore[override] class BaseListProxy(BaseProxy, MutableSequence[_T]): __builtins__: ClassVar[dict[str, Any]] diff --git a/mypy/typeshed/stdlib/multiprocessing/popen_forkserver.pyi b/mypy/typeshed/stdlib/multiprocessing/popen_forkserver.pyi index d28c7245fd54..f7d53bbb3e41 100644 --- a/mypy/typeshed/stdlib/multiprocessing/popen_forkserver.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/popen_forkserver.pyi @@ -1,5 +1,4 @@ import sys -from multiprocessing.process import BaseProcess from typing import ClassVar from . import popen_fork @@ -15,8 +14,3 @@ if sys.platform != "win32": class Popen(popen_fork.Popen): DupFd: ClassVar[type[_DupFd]] finalizer: Finalize - sentinel: int - - def __init__(self, process_obj: BaseProcess) -> None: ... - def duplicate_for_child(self, fd: int) -> int: ... - def poll(self, flag: int = ...) -> int | None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/popen_spawn_posix.pyi b/mypy/typeshed/stdlib/multiprocessing/popen_spawn_posix.pyi index 81aaac7ca459..7e81d39600ad 100644 --- a/mypy/typeshed/stdlib/multiprocessing/popen_spawn_posix.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/popen_spawn_posix.pyi @@ -1,5 +1,4 @@ import sys -from multiprocessing.process import BaseProcess from typing import ClassVar from . import popen_fork @@ -19,6 +18,3 @@ if sys.platform != "win32": finalizer: Finalize pid: int # may not exist if _launch raises in second try / except sentinel: int # may not exist if _launch raises in second try / except - - def __init__(self, process_obj: BaseProcess) -> None: ... - def duplicate_for_child(self, fd: int) -> int: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/queues.pyi b/mypy/typeshed/stdlib/multiprocessing/queues.pyi index 1d31fa694c45..02a67216c72b 100644 --- a/mypy/typeshed/stdlib/multiprocessing/queues.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/queues.pyi @@ -15,18 +15,13 @@ class Queue(queue.Queue[_T]): def __init__(self, maxsize: int = ..., *, ctx: Any = ...) -> None: ... def get(self, block: bool = ..., timeout: float | None = ...) -> _T: ... def put(self, obj: _T, block: bool = ..., timeout: float | None = ...) -> None: ... - def qsize(self) -> int: ... - def empty(self) -> bool: ... - def full(self) -> bool: ... def put_nowait(self, item: _T) -> None: ... def get_nowait(self) -> _T: ... def close(self) -> None: ... def join_thread(self) -> None: ... def cancel_join_thread(self) -> None: ... -class JoinableQueue(Queue[_T]): - def task_done(self) -> None: ... - def join(self) -> None: ... +class JoinableQueue(Queue[_T]): ... class SimpleQueue(Generic[_T]): def __init__(self, *, ctx: Any = ...) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi index a22c16828780..cab86d866bab 100644 --- a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi @@ -1,10 +1,9 @@ import pickle import sys -from _typeshed import HasFileno +from _typeshed import HasFileno, Incomplete from abc import ABCMeta from copyreg import _DispatchTableType from socket import socket -from typing import Any from typing_extensions import Literal if sys.platform == "win32": @@ -18,12 +17,12 @@ class ForkingPickler(pickle.Pickler): @classmethod def register(cls, type, reduce) -> None: ... @classmethod - def dumps(cls, obj, protocol: Any | None = ...): ... + def dumps(cls, obj, protocol: Incomplete | None = ...): ... loads = pickle.loads register = ForkingPickler.register -def dump(obj, file, protocol: Any | None = ...) -> None: ... +def dump(obj, file, protocol: Incomplete | None = ...) -> None: ... if sys.platform == "win32": if sys.version_info >= (3, 8): @@ -38,7 +37,7 @@ if sys.platform == "win32": def recv_handle(conn): ... class DupHandle: - def __init__(self, handle, access, pid: Any | None = ...) -> None: ... + def __init__(self, handle, access, pid: Incomplete | None = ...) -> None: ... def detach(self): ... else: diff --git a/mypy/typeshed/stdlib/multiprocessing/resource_tracker.pyi b/mypy/typeshed/stdlib/multiprocessing/resource_tracker.pyi index 98abb075fb3d..50f3db67467b 100644 --- a/mypy/typeshed/stdlib/multiprocessing/resource_tracker.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/resource_tracker.pyi @@ -4,7 +4,6 @@ from collections.abc import Sized __all__ = ["ensure_running", "register", "unregister"] class ResourceTracker: - def __init__(self) -> None: ... def getfd(self) -> int | None: ... def ensure_running(self) -> None: ... def register(self, name: Sized, rtype: Incomplete) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/util.pyi b/mypy/typeshed/stdlib/multiprocessing/util.pyi index e89b4a71cad4..4b93b7a6a472 100644 --- a/mypy/typeshed/stdlib/multiprocessing/util.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/util.pyi @@ -69,12 +69,10 @@ def is_exiting() -> bool: ... class ForkAwareThreadLock: acquire: Callable[[bool, float], bool] release: Callable[[], None] - def __init__(self) -> None: ... def __enter__(self) -> bool: ... def __exit__(self, *args: object) -> None: ... -class ForkAwareLocal(threading.local): - def __init__(self) -> None: ... +class ForkAwareLocal(threading.local): ... MAXFD: int diff --git a/mypy/typeshed/stdlib/optparse.pyi b/mypy/typeshed/stdlib/optparse.pyi index b571ff0680b7..5cff39717a7b 100644 --- a/mypy/typeshed/stdlib/optparse.pyi +++ b/mypy/typeshed/stdlib/optparse.pyi @@ -42,7 +42,6 @@ class AmbiguousOptionError(BadOptionError): def __init__(self, opt_str: str, possibilities: Sequence[str]) -> None: ... class OptionError(OptParseError): - msg: str option_id: str def __init__(self, msg: str, option: Option) -> None: ... diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index e3d428555462..6f51d4e7aa50 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -9,9 +9,12 @@ from _typeshed import ( OpenBinaryModeUpdating, OpenBinaryModeWriting, OpenTextMode, + ReadableBuffer, Self, StrOrBytesPath, StrPath, + SupportsLenAndGetItem, + WriteableBuffer, structseq, ) from abc import abstractmethod @@ -604,7 +607,6 @@ def pipe() -> tuple[int, int]: ... def read(__fd: int, __length: int) -> bytes: ... if sys.platform != "win32": - # Unix only def fchmod(fd: int, mode: int) -> None: ... def fchown(fd: int, uid: int, gid: int) -> None: ... def fpathconf(__fd: int, __name: str | int) -> int: ... @@ -621,11 +623,12 @@ if sys.platform != "win32": def pread(__fd: int, __length: int, __offset: int) -> bytes: ... def pwrite(__fd: int, __buffer: bytes, __offset: int) -> int: ... + # In CI, stubtest sometimes reports that these are available on MacOS, sometimes not + def preadv(__fd: int, __buffers: SupportsLenAndGetItem[WriteableBuffer], __offset: int, __flags: int = ...) -> int: ... + def pwritev(__fd: int, __buffers: SupportsLenAndGetItem[ReadableBuffer], __offset: int, __flags: int = ...) -> int: ... if sys.platform != "darwin": if sys.version_info >= (3, 10): RWF_APPEND: int # docs say available on 3.7+, stubtest says otherwise - def preadv(__fd: int, __buffers: Iterable[bytes], __offset: int, __flags: int = ...) -> int: ... - def pwritev(__fd: int, __buffers: Iterable[bytes], __offset: int, __flags: int = ...) -> int: ... RWF_DSYNC: int RWF_SYNC: int RWF_HIPRI: int @@ -642,8 +645,8 @@ if sys.platform != "win32": trailers: Sequence[bytes] = ..., flags: int = ..., ) -> int: ... # FreeBSD and Mac OS X only - def readv(__fd: int, __buffers: Sequence[bytearray]) -> int: ... - def writev(__fd: int, __buffers: Sequence[bytes]) -> int: ... + def readv(__fd: int, __buffers: SupportsLenAndGetItem[WriteableBuffer]) -> int: ... + def writev(__fd: int, __buffers: SupportsLenAndGetItem[ReadableBuffer]) -> int: ... @final class terminal_size(structseq[int], tuple[int, int]): diff --git a/mypy/typeshed/stdlib/pipes.pyi b/mypy/typeshed/stdlib/pipes.pyi index d6bbd7eafac3..fe680bfddf5f 100644 --- a/mypy/typeshed/stdlib/pipes.pyi +++ b/mypy/typeshed/stdlib/pipes.pyi @@ -3,7 +3,6 @@ import os __all__ = ["Template"] class Template: - def __init__(self) -> None: ... def reset(self) -> None: ... def clone(self) -> Template: ... def debug(self, flag: bool) -> None: ... diff --git a/mypy/typeshed/stdlib/posix.pyi b/mypy/typeshed/stdlib/posix.pyi index 7055f15f3d67..ffd96757586b 100644 --- a/mypy/typeshed/stdlib/posix.pyi +++ b/mypy/typeshed/stdlib/posix.pyi @@ -309,17 +309,10 @@ if sys.platform != "win32": copy_file_range as copy_file_range, memfd_create as memfd_create, ) - from os import register_at_fork as register_at_fork + from os import preadv as preadv, pwritev as pwritev, register_at_fork as register_at_fork if sys.platform != "darwin": - from os import ( - RWF_DSYNC as RWF_DSYNC, - RWF_HIPRI as RWF_HIPRI, - RWF_NOWAIT as RWF_NOWAIT, - RWF_SYNC as RWF_SYNC, - preadv as preadv, - pwritev as pwritev, - ) + from os import RWF_DSYNC as RWF_DSYNC, RWF_HIPRI as RWF_HIPRI, RWF_NOWAIT as RWF_NOWAIT, RWF_SYNC as RWF_SYNC # Not same as os.environ or os.environb # Because of this variable, we can't do "from posix import *" in os/__init__.pyi diff --git a/mypy/typeshed/stdlib/pstats.pyi b/mypy/typeshed/stdlib/pstats.pyi index 7629cd63438f..10d817b59630 100644 --- a/mypy/typeshed/stdlib/pstats.pyi +++ b/mypy/typeshed/stdlib/pstats.pyi @@ -30,7 +30,7 @@ if sys.version_info >= (3, 9): @dataclass(unsafe_hash=True) class FunctionProfile: - ncalls: int + ncalls: str tottime: float percall_tottime: float cumtime: float diff --git a/mypy/typeshed/stdlib/pydoc.pyi b/mypy/typeshed/stdlib/pydoc.pyi index abcffc31111a..0dd2739797f9 100644 --- a/mypy/typeshed/stdlib/pydoc.pyi +++ b/mypy/typeshed/stdlib/pydoc.pyi @@ -6,6 +6,7 @@ from collections.abc import Callable, Container, Mapping, MutableMapping from reprlib import Repr from types import MethodType, ModuleType, TracebackType from typing import IO, Any, AnyStr, NoReturn, TypeVar +from typing_extensions import TypeGuard __all__ = ["help"] @@ -60,12 +61,6 @@ class Doc: def getdocloc(self, object: object, basedir: str = ...) -> str | None: ... class HTMLRepr(Repr): - maxlist: int - maxtuple: int - maxdict: int - maxstring: int - maxother: int - def __init__(self) -> None: ... def escape(self, text: str) -> str: ... def repr(self, object: object) -> str: ... def repr1(self, x: object, level: complex) -> str: ... @@ -153,12 +148,6 @@ class HTMLDoc(Doc): def filelink(self, url: str, path: str) -> str: ... class TextRepr(Repr): - maxlist: int - maxtuple: int - maxdict: int - maxstring: int - maxother: int - def __init__(self) -> None: ... def repr1(self, x: object, level: complex) -> str: ... def repr_string(self, x: str, level: complex) -> str: ... def repr_str(self, x: str, level: complex) -> str: ... @@ -243,5 +232,5 @@ class ModuleScanner: ) -> None: ... def apropos(key: str) -> None: ... -def ispath(x: Any) -> bool: ... +def ispath(x: object) -> TypeGuard[str]: ... def cli() -> None: ... diff --git a/mypy/typeshed/stdlib/random.pyi b/mypy/typeshed/stdlib/random.pyi index 3bb999bfaaa6..a2a1d956e78f 100644 --- a/mypy/typeshed/stdlib/random.pyi +++ b/mypy/typeshed/stdlib/random.pyi @@ -50,7 +50,6 @@ class Random(_random.Random): def getstate(self) -> tuple[Any, ...]: ... def setstate(self, state: tuple[Any, ...]) -> None: ... - def getrandbits(self, __k: int) -> int: ... def randrange(self, start: int, stop: int | None = ..., step: int = ...) -> int: ... def randint(self, a: int, b: int) -> int: ... if sys.version_info >= (3, 9): @@ -78,7 +77,6 @@ class Random(_random.Random): else: def sample(self, population: Sequence[_T] | AbstractSet[_T], k: int) -> list[_T]: ... - def random(self) -> float: ... def uniform(self, a: float, b: float) -> float: ... def triangular(self, low: float = ..., high: float = ..., mode: float | None = ...) -> float: ... def betavariate(self, alpha: float, beta: float) -> float: ... diff --git a/mypy/typeshed/stdlib/reprlib.pyi b/mypy/typeshed/stdlib/reprlib.pyi index d5554344c494..9955f12627a3 100644 --- a/mypy/typeshed/stdlib/reprlib.pyi +++ b/mypy/typeshed/stdlib/reprlib.pyi @@ -22,7 +22,6 @@ class Repr: maxlong: int maxstring: int maxother: int - def __init__(self) -> None: ... def repr(self, x: Any) -> str: ... def repr1(self, x: Any, level: int) -> str: ... def repr_tuple(self, x: tuple[Any, ...], level: int) -> str: ... diff --git a/mypy/typeshed/stdlib/select.pyi b/mypy/typeshed/stdlib/select.pyi index 7cfea9ea0fc1..63989730a7e9 100644 --- a/mypy/typeshed/stdlib/select.pyi +++ b/mypy/typeshed/stdlib/select.pyi @@ -21,7 +21,6 @@ if sys.platform != "win32": POLLWRNORM: int class poll: - def __init__(self) -> None: ... def register(self, fd: FileDescriptorLike, eventmask: int = ...) -> None: ... def modify(self, fd: FileDescriptorLike, eventmask: int) -> None: ... def unregister(self, fd: FileDescriptorLike) -> None: ... diff --git a/mypy/typeshed/stdlib/smtplib.pyi b/mypy/typeshed/stdlib/smtplib.pyi index c42841c43e7f..2d03b60e7bb4 100644 --- a/mypy/typeshed/stdlib/smtplib.pyi +++ b/mypy/typeshed/stdlib/smtplib.pyi @@ -49,7 +49,6 @@ class SMTPResponseException(SMTPException): def __init__(self, code: int, msg: bytes | str) -> None: ... class SMTPSenderRefused(SMTPResponseException): - smtp_code: int smtp_error: bytes sender: str args: tuple[int, bytes, str] @@ -151,7 +150,6 @@ class SMTP: def quit(self) -> _Reply: ... class SMTP_SSL(SMTP): - default_port: int keyfile: str | None certfile: str | None context: SSLContext diff --git a/mypy/typeshed/stdlib/socket.pyi b/mypy/typeshed/stdlib/socket.pyi index a0f5708bf806..89a6d059f165 100644 --- a/mypy/typeshed/stdlib/socket.pyi +++ b/mypy/typeshed/stdlib/socket.pyi @@ -297,6 +297,20 @@ if sys.platform == "linux": CAN_RAW_RECV_OWN_MSGS as CAN_RAW_RECV_OWN_MSGS, CAN_RTR_FLAG as CAN_RTR_FLAG, CAN_SFF_MASK as CAN_SFF_MASK, + NETLINK_ARPD as NETLINK_ARPD, + NETLINK_CRYPTO as NETLINK_CRYPTO, + NETLINK_DNRTMSG as NETLINK_DNRTMSG, + NETLINK_FIREWALL as NETLINK_FIREWALL, + NETLINK_IP6_FW as NETLINK_IP6_FW, + NETLINK_NFLOG as NETLINK_NFLOG, + NETLINK_ROUTE as NETLINK_ROUTE, + NETLINK_ROUTE6 as NETLINK_ROUTE6, + NETLINK_SKIP as NETLINK_SKIP, + NETLINK_TAPBASE as NETLINK_TAPBASE, + NETLINK_TCPDIAG as NETLINK_TCPDIAG, + NETLINK_USERSOCK as NETLINK_USERSOCK, + NETLINK_W1 as NETLINK_W1, + NETLINK_XFRM as NETLINK_XFRM, PACKET_BROADCAST as PACKET_BROADCAST, PACKET_FASTROUTE as PACKET_FASTROUTE, PACKET_HOST as PACKET_HOST, diff --git a/mypy/typeshed/stdlib/socketserver.pyi b/mypy/typeshed/stdlib/socketserver.pyi index f1d127ebe6a1..e597818ef7da 100644 --- a/mypy/typeshed/stdlib/socketserver.pyi +++ b/mypy/typeshed/stdlib/socketserver.pyi @@ -70,8 +70,8 @@ class BaseServer: def close_request(self, request: _RequestType) -> None: ... # undocumented class TCPServer(BaseServer): - allow_reuse_port: bool - request_queue_size: int + if sys.version_info >= (3, 11): + allow_reuse_port: bool def __init__( self: Self, server_address: tuple[str, int], @@ -80,11 +80,9 @@ class TCPServer(BaseServer): ) -> None: ... def get_request(self) -> tuple[_socket, Any]: ... -class UDPServer(BaseServer): - if sys.version_info >= (3, 11): - allow_reuse_port: bool +class UDPServer(TCPServer): max_packet_size: ClassVar[int] - def get_request(self) -> tuple[tuple[bytes, _socket], Any]: ... + def get_request(self) -> tuple[tuple[bytes, _socket], Any]: ... # type: ignore[override] if sys.platform != "win32": class UnixStreamServer(BaseServer): diff --git a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi index 83d2df1e6da9..189e796de109 100644 --- a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi +++ b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi @@ -217,7 +217,7 @@ def enable_callback_tracebacks(__enable: bool) -> None: ... # takes a pos-or-keyword argument because there is a C wrapper def enable_shared_cache(enable: int) -> None: ... -if sys.version_info >= (3, 11): +if sys.version_info >= (3, 10): def register_adapter(__type: type[_T], __adapter: _Adapter[_T]) -> None: ... def register_converter(__typename: str, __converter: _Converter) -> None: ... @@ -318,10 +318,10 @@ class Connection: def create_collation(self, __name: str, __callback: Callable[[str, str], int | SupportsIndex] | None) -> None: ... if sys.version_info >= (3, 8): def create_function( - self, name: str, narg: int, func: Callable[..., _SqliteData], *, deterministic: bool = ... + self, name: str, narg: int, func: Callable[..., _SqliteData] | None, *, deterministic: bool = ... ) -> None: ... else: - def create_function(self, name: str, num_params: int, func: Callable[..., _SqliteData]) -> None: ... + def create_function(self, name: str, num_params: int, func: Callable[..., _SqliteData] | None) -> None: ... @overload def cursor(self, cursorClass: None = ...) -> Cursor: ... diff --git a/mypy/typeshed/stdlib/sre_parse.pyi b/mypy/typeshed/stdlib/sre_parse.pyi index e4d66d1baf52..3dcf8ad78dee 100644 --- a/mypy/typeshed/stdlib/sre_parse.pyi +++ b/mypy/typeshed/stdlib/sre_parse.pyi @@ -27,7 +27,6 @@ class _State: groupdict: dict[str, int] groupwidths: list[int | None] lookbehindgroups: int | None - def __init__(self) -> None: ... @property def groups(self) -> int: ... def opengroup(self, name: str | None = ...) -> int: ... diff --git a/mypy/typeshed/stdlib/ssl.pyi b/mypy/typeshed/stdlib/ssl.pyi index 09c8d07780a7..6443a6ea61ba 100644 --- a/mypy/typeshed/stdlib/ssl.pyi +++ b/mypy/typeshed/stdlib/ssl.pyi @@ -356,7 +356,6 @@ class SSLContext: keylog_filename: str post_handshake_auth: bool def __new__(cls: type[Self], protocol: int = ..., *args: Any, **kwargs: Any) -> Self: ... - def __init__(self, protocol: int = ...) -> None: ... def cert_store_stats(self) -> dict[str, int]: ... def load_cert_chain( self, certfile: StrOrBytesPath, keyfile: StrOrBytesPath | None = ..., password: _PasswordType | None = ... diff --git a/mypy/typeshed/stdlib/string.pyi b/mypy/typeshed/stdlib/string.pyi index 1b9ba5b58fa1..6fb803fe53be 100644 --- a/mypy/typeshed/stdlib/string.pyi +++ b/mypy/typeshed/stdlib/string.pyi @@ -2,8 +2,8 @@ import sys from _typeshed import StrOrLiteralStr from collections.abc import Iterable, Mapping, Sequence from re import Pattern, RegexFlag -from typing import Any, overload -from typing_extensions import LiteralString +from typing import Any, ClassVar, overload +from typing_extensions import LiteralString, TypeAlias __all__ = [ "ascii_letters", @@ -32,13 +32,20 @@ whitespace: LiteralString def capwords(s: StrOrLiteralStr, sep: StrOrLiteralStr | None = ...) -> StrOrLiteralStr: ... -class Template: +if sys.version_info >= (3, 9): + _TemplateMetaclass: TypeAlias = type +else: + class _TemplateMetaclass(type): + pattern: ClassVar[str] + def __init__(cls, name: str, bases: tuple[type, ...], dct: dict[str, Any]) -> None: ... + +class Template(metaclass=_TemplateMetaclass): template: str - delimiter: str - idpattern: str - braceidpattern: str | None - flags: RegexFlag - pattern: Pattern[str] + delimiter: ClassVar[str] + idpattern: ClassVar[str] + braceidpattern: ClassVar[str | None] + flags: ClassVar[RegexFlag] + pattern: ClassVar[Pattern[str]] def __init__(self, template: str) -> None: ... def substitute(self, __mapping: Mapping[str, object] = ..., **kwds: object) -> str: ... def safe_substitute(self, __mapping: Mapping[str, object] = ..., **kwds: object) -> str: ... diff --git a/mypy/typeshed/stdlib/struct.pyi b/mypy/typeshed/stdlib/struct.pyi index f7eff2b76f14..74afddd74262 100644 --- a/mypy/typeshed/stdlib/struct.pyi +++ b/mypy/typeshed/stdlib/struct.pyi @@ -14,8 +14,10 @@ def iter_unpack(__format: str | bytes, __buffer: ReadableBuffer) -> Iterator[tup def calcsize(__format: str | bytes) -> int: ... class Struct: - format: str - size: int + @property + def format(self) -> str: ... + @property + def size(self) -> int: ... def __init__(self, format: str | bytes) -> None: ... def pack(self, *v: Any) -> bytes: ... def pack_into(self, buffer: WriteableBuffer, offset: int, *v: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/subprocess.pyi b/mypy/typeshed/stdlib/subprocess.pyi index fded3f74928e..25b988adc52d 100644 --- a/mypy/typeshed/stdlib/subprocess.pyi +++ b/mypy/typeshed/stdlib/subprocess.pyi @@ -1828,8 +1828,8 @@ class TimeoutExpired(SubprocessError): timeout: float # morally: _TXT | None output: Any - stdout: Any - stderr: Any + stdout: bytes | None + stderr: bytes | None class CalledProcessError(SubprocessError): returncode: int diff --git a/mypy/typeshed/stdlib/symtable.pyi b/mypy/typeshed/stdlib/symtable.pyi index d44b2d7927b3..98b62edbfc6a 100644 --- a/mypy/typeshed/stdlib/symtable.pyi +++ b/mypy/typeshed/stdlib/symtable.pyi @@ -59,6 +59,5 @@ class Symbol: def get_namespace(self) -> SymbolTable: ... class SymbolTableFactory: - def __init__(self) -> None: ... def new(self, table: Any, filename: str) -> SymbolTable: ... def __call__(self, table: Any, filename: str) -> SymbolTable: ... diff --git a/mypy/typeshed/stdlib/sys.pyi b/mypy/typeshed/stdlib/sys.pyi index a1c875561a87..c3747235d628 100644 --- a/mypy/typeshed/stdlib/sys.pyi +++ b/mypy/typeshed/stdlib/sys.pyi @@ -11,6 +11,8 @@ from typing_extensions import Literal, TypeAlias, final _T = TypeVar("_T") +# see https://github.com/python/typeshed/issues/8513#issue-1333671093 for the rationale behind this alias +_ExitCode: TypeAlias = str | int | None _OptExcInfo: TypeAlias = OptExcInfo # noqa: Y047 # TODO: obsolete, remove fall 2022 or later # Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder` @@ -188,11 +190,15 @@ class _implementation: int_info: _int_info @final -class _int_info(structseq[int], tuple[int, int]): +class _int_info(structseq[int], tuple[int, int, int, int]): @property def bits_per_digit(self) -> int: ... @property def sizeof_digit(self) -> int: ... + @property + def default_max_str_digits(self) -> int: ... + @property + def str_digits_check_threshold(self) -> int: ... @final class _version_info(_UninstantiableStructseq, tuple[int, int, int, str, int]): @@ -221,8 +227,7 @@ def exc_info() -> OptExcInfo: ... if sys.version_info >= (3, 11): def exception() -> BaseException | None: ... -# sys.exit() accepts an optional argument of anything printable -def exit(__status: object = ...) -> NoReturn: ... +def exit(__status: _ExitCode = ...) -> NoReturn: ... def getallocatedblocks() -> int: ... def getdefaultencoding() -> str: ... @@ -327,3 +332,8 @@ if sys.version_info < (3, 8): _CoroWrapper: TypeAlias = Callable[[Coroutine[Any, Any, Any]], Any] def set_coroutine_wrapper(__wrapper: _CoroWrapper) -> None: ... def get_coroutine_wrapper() -> _CoroWrapper: ... + +# The following two functions were added in 3.11.0, 3.10.7, 3.9.14, 3.8.14, & 3.7.14, +# as part of the response to CVE-2020-10735 +def set_int_max_str_digits(maxdigits: int) -> None: ... +def get_int_max_str_digits() -> int: ... diff --git a/mypy/typeshed/stdlib/sysconfig.pyi b/mypy/typeshed/stdlib/sysconfig.pyi index 03362b5caef9..895abc2cd047 100644 --- a/mypy/typeshed/stdlib/sysconfig.pyi +++ b/mypy/typeshed/stdlib/sysconfig.pyi @@ -16,7 +16,7 @@ __all__ = [ "parse_config_h", ] -def get_config_var(name: str) -> str | None: ... +def get_config_var(name: str) -> Any: ... @overload def get_config_vars() -> dict[str, Any]: ... @overload diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index cf74899a8fb4..8855e1a953db 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -6,7 +6,7 @@ from builtins import list as _list, type as Type # aliases to avoid name clashe from collections.abc import Callable, Iterable, Iterator, Mapping from gzip import _ReadableFileobj as _GzipReadableFileobj, _WritableFileobj as _GzipWritableFileobj from types import TracebackType -from typing import IO, Protocol, overload +from typing import IO, ClassVar, Protocol, overload from typing_extensions import Literal __all__ = [ @@ -110,7 +110,7 @@ class ExFileObject(io.BufferedReader): def __init__(self, tarfile: TarFile, tarinfo: TarInfo) -> None: ... class TarFile: - OPEN_METH: Mapping[str, str] + OPEN_METH: ClassVar[Mapping[str, str]] name: StrOrBytesPath | None mode: Literal["r", "a", "w", "x"] fileobj: _Fileobj | None diff --git a/mypy/typeshed/stdlib/threading.pyi b/mypy/typeshed/stdlib/threading.pyi index 289a86826ecd..6fb1ab99c833 100644 --- a/mypy/typeshed/stdlib/threading.pyi +++ b/mypy/typeshed/stdlib/threading.pyi @@ -102,7 +102,6 @@ class _DummyThread(Thread): def __init__(self) -> None: ... class Lock: - def __init__(self) -> None: ... def __enter__(self) -> bool: ... def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None @@ -112,7 +111,6 @@ class Lock: def locked(self) -> bool: ... class _RLock: - def __init__(self) -> None: ... def acquire(self, blocking: bool = ..., timeout: float = ...) -> bool: ... def release(self) -> None: ... __enter__ = acquire @@ -148,7 +146,6 @@ class Semaphore: class BoundedSemaphore(Semaphore): ... class Event: - def __init__(self) -> None: ... def is_set(self) -> bool: ... def isSet(self) -> bool: ... # deprecated alias for is_set() def set(self) -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index d8dd463b5a8c..699dfd2a408a 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -178,13 +178,11 @@ _ButtonCommand: TypeAlias = str | Callable[[], Any] # accepts string of tcl cod _CanvasItemId: TypeAlias = int _Color: TypeAlias = str # typically '#rrggbb', '#rgb' or color names. _Compound: TypeAlias = Literal["top", "left", "center", "right", "bottom", "none"] # -compound in manual page named 'options' -_Cursor: TypeAlias = Union[ - str, tuple[str], tuple[str, str], tuple[str, str, str], tuple[str, str, str, str] -] # manual page: Tk_GetCursor -_EntryValidateCommand: TypeAlias = ( - str | list[str] | tuple[str, ...] | Callable[[], bool] -) # example when it's sequence: entry['invalidcommand'] = [entry.register(print), '%P'] -_GridIndex: TypeAlias = int | str | Literal["all"] +# manual page: Tk_GetCursor +_Cursor: TypeAlias = Union[str, tuple[str], tuple[str, str], tuple[str, str, str], tuple[str, str, str, str]] +# example when it's sequence: entry['invalidcommand'] = [entry.register(print), '%P'] +_EntryValidateCommand: TypeAlias = str | list[str] | tuple[str, ...] | Callable[[], bool] +_GridIndex: TypeAlias = int | str _ImageSpec: TypeAlias = _Image | str # str can be from e.g. tkinter.image_names() _Relief: TypeAlias = Literal["raised", "sunken", "flat", "ridge", "solid", "groove"] # manual page: Tk_GetRelief _ScreenUnits: TypeAlias = str | float # Often the right type instead of int. Manual page: Tk_GetPixels diff --git a/mypy/typeshed/stdlib/tkinter/colorchooser.pyi b/mypy/typeshed/stdlib/tkinter/colorchooser.pyi index ac2ea187bdd5..47eb222590c6 100644 --- a/mypy/typeshed/stdlib/tkinter/colorchooser.pyi +++ b/mypy/typeshed/stdlib/tkinter/colorchooser.pyi @@ -1,4 +1,5 @@ import sys +from tkinter import Misc, _Color from tkinter.commondialog import Dialog from typing import ClassVar @@ -8,4 +9,12 @@ if sys.version_info >= (3, 9): class Chooser(Dialog): command: ClassVar[str] -def askcolor(color: str | bytes | None = ..., **options) -> tuple[None, None] | tuple[tuple[float, float, float], str]: ... +if sys.version_info >= (3, 9): + def askcolor( + color: str | bytes | None = ..., *, initialcolor: _Color = ..., parent: Misc = ..., title: str = ... + ) -> tuple[None, None] | tuple[tuple[int, int, int], str]: ... + +else: + def askcolor( + color: str | bytes | None = ..., *, initialcolor: _Color = ..., parent: Misc = ..., title: str = ... + ) -> tuple[None, None] | tuple[tuple[float, float, float], str]: ... diff --git a/mypy/typeshed/stdlib/tkinter/ttk.pyi b/mypy/typeshed/stdlib/tkinter/ttk.pyi index a191b3be281a..07584ed9ed87 100644 --- a/mypy/typeshed/stdlib/tkinter/ttk.pyi +++ b/mypy/typeshed/stdlib/tkinter/ttk.pyi @@ -937,7 +937,7 @@ class _TreeviewTagDict(TypedDict): foreground: tkinter._Color background: tkinter._Color font: _FontDescription - image: Literal[""] | str # not wrapped in list :D + image: str # not wrapped in list :D class _TreeviewHeaderDict(TypedDict): text: str @@ -963,7 +963,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): class_: str = ..., columns: str | list[str] | tuple[str, ...] = ..., cursor: tkinter._Cursor = ..., - displaycolumns: str | list[str] | tuple[str, ...] | list[int] | tuple[int, ...] | Literal["#all"] = ..., + displaycolumns: str | list[str] | tuple[str, ...] | list[int] | tuple[int, ...] = ..., height: int = ..., name: str = ..., padding: _Padding = ..., @@ -985,7 +985,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): *, columns: str | list[str] | tuple[str, ...] = ..., cursor: tkinter._Cursor = ..., - displaycolumns: str | list[str] | tuple[str, ...] | list[int] | tuple[int, ...] | Literal["#all"] = ..., + displaycolumns: str | list[str] | tuple[str, ...] | list[int] | tuple[int, ...] = ..., height: int = ..., padding: _Padding = ..., selectmode: Literal["extended", "browse", "none"] = ..., diff --git a/mypy/typeshed/stdlib/tokenize.pyi b/mypy/typeshed/stdlib/tokenize.pyi index 1a67736e78de..6f242a6cd1ef 100644 --- a/mypy/typeshed/stdlib/tokenize.pyi +++ b/mypy/typeshed/stdlib/tokenize.pyi @@ -115,7 +115,6 @@ class Untokenizer: prev_row: int prev_col: int encoding: str | None - def __init__(self) -> None: ... def add_whitespace(self, start: _Position) -> None: ... def untokenize(self, iterable: Iterable[_Token]) -> str: ... def compat(self, token: Sequence[int | str], iterable: Iterable[_Token]) -> None: ... diff --git a/mypy/typeshed/stdlib/traceback.pyi b/mypy/typeshed/stdlib/traceback.pyi index fcaa39bf42f7..bf8e24e7ab27 100644 --- a/mypy/typeshed/stdlib/traceback.pyi +++ b/mypy/typeshed/stdlib/traceback.pyi @@ -92,13 +92,12 @@ else: def format_exc(limit: int | None = ..., chain: bool = ...) -> str: ... def format_tb(tb: TracebackType | None, limit: int | None = ...) -> list[str]: ... def format_stack(f: FrameType | None = ..., limit: int | None = ...) -> list[str]: ... -def clear_frames(tb: TracebackType) -> None: ... +def clear_frames(tb: TracebackType | None) -> None: ... def walk_stack(f: FrameType | None) -> Iterator[tuple[FrameType, int]]: ... def walk_tb(tb: TracebackType | None) -> Iterator[tuple[FrameType, int]]: ... if sys.version_info >= (3, 11): class _ExceptionPrintContext: - def __init__(self) -> None: ... def indent(self) -> str: ... def emit(self, text_gen: str | Iterable[str], margin_char: str | None = ...) -> Generator[str, None, None]: ... diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 28fce697f2ca..16fe096d3117 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -304,7 +304,7 @@ class CodeType: class MappingProxyType(Mapping[_KT, _VT_co], Generic[_KT, _VT_co]): __hash__: ClassVar[None] # type: ignore[assignment] def __init__(self, mapping: SupportsKeysAndGetItem[_KT, _VT_co]) -> None: ... - def __getitem__(self, __k: _KT) -> _VT_co: ... + def __getitem__(self, __key: _KT) -> _VT_co: ... def __iter__(self) -> Iterator[_KT]: ... def __len__(self) -> int: ... def copy(self) -> dict[_KT, _VT_co]: ... @@ -344,12 +344,6 @@ class ModuleType: @final class GeneratorType(Generator[_T_co, _T_contra, _V_co]): - @property - def gi_code(self) -> CodeType: ... - @property - def gi_frame(self) -> FrameType: ... - @property - def gi_running(self) -> bool: ... @property def gi_yieldfrom(self) -> GeneratorType[_T_co, _T_contra, Any] | None: ... if sys.version_info >= (3, 11): @@ -359,7 +353,6 @@ class GeneratorType(Generator[_T_co, _T_contra, _V_co]): __qualname__: str def __iter__(self) -> GeneratorType[_T_co, _T_contra, _V_co]: ... def __next__(self) -> _T_co: ... - def close(self) -> None: ... def send(self, __arg: _T_contra) -> _T_co: ... @overload def throw( @@ -372,12 +365,6 @@ class GeneratorType(Generator[_T_co, _T_contra, _V_co]): class AsyncGeneratorType(AsyncGenerator[_T_co, _T_contra]): @property def ag_await(self) -> Awaitable[Any] | None: ... - @property - def ag_frame(self) -> FrameType: ... - @property - def ag_running(self) -> bool: ... - @property - def ag_code(self) -> CodeType: ... __name__: str __qualname__: str def __aiter__(self) -> AsyncGeneratorType[_T_co, _T_contra]: ... @@ -398,14 +385,6 @@ class CoroutineType(Coroutine[_T_co, _T_contra, _V_co]): __name__: str __qualname__: str @property - def cr_await(self) -> Any | None: ... - @property - def cr_code(self) -> CodeType: ... - @property - def cr_frame(self) -> FrameType: ... - @property - def cr_running(self) -> bool: ... - @property def cr_origin(self) -> tuple[tuple[str, int, str], ...] | None: ... if sys.version_info >= (3, 11): @property diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index a186bb92bf00..954f47d14502 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -1,6 +1,7 @@ import _typeshed import collections # Needed by aliases like DefaultDict, see mypy issue 2986 import sys +from _collections_abc import dict_items, dict_keys, dict_values from _typeshed import IdentityFunction, Incomplete, SupportsKeysAndGetItem from abc import ABCMeta, abstractmethod from contextlib import AbstractAsyncContextManager, AbstractContextManager @@ -17,7 +18,7 @@ from types import ( TracebackType, WrapperDescriptorType, ) -from typing_extensions import ParamSpec as _ParamSpec, final as _final +from typing_extensions import Never as _Never, ParamSpec as _ParamSpec, final as _final __all__ = [ "AbstractSet", @@ -446,6 +447,7 @@ class AsyncGenerator(AsyncIterator[_T_co], Generic[_T_co, _T_contra]): @runtime_checkable class Container(Protocol[_T_co]): + # This is generic more on vibes than anything else @abstractmethod def __contains__(self, __x: object) -> bool: ... @@ -575,7 +577,7 @@ class Mapping(Collection[_KT], Generic[_KT, _VT_co]): # TODO: We wish the key type could also be covariant, but that doesn't work, # see discussion in https://github.com/python/typing/pull/273. @abstractmethod - def __getitem__(self, __k: _KT) -> _VT_co: ... + def __getitem__(self, __key: _KT) -> _VT_co: ... # Mixin methods @overload def get(self, __key: _KT) -> _VT_co | None: ... @@ -588,9 +590,9 @@ class Mapping(Collection[_KT], Generic[_KT, _VT_co]): class MutableMapping(Mapping[_KT, _VT], Generic[_KT, _VT]): @abstractmethod - def __setitem__(self, __k: _KT, __v: _VT) -> None: ... + def __setitem__(self, __key: _KT, __value: _VT) -> None: ... @abstractmethod - def __delitem__(self, __v: _KT) -> None: ... + def __delitem__(self, __key: _KT) -> None: ... def clear(self) -> None: ... @overload def pop(self, __key: _KT) -> _VT: ... @@ -790,16 +792,16 @@ class _TypedDict(Mapping[str, object], metaclass=ABCMeta): __required_keys__: ClassVar[frozenset[str]] __optional_keys__: ClassVar[frozenset[str]] def copy(self: _typeshed.Self) -> _typeshed.Self: ... - # Using NoReturn so that only calls using mypy plugin hook that specialize the signature + # Using Never so that only calls using mypy plugin hook that specialize the signature # can go through. - def setdefault(self, k: NoReturn, default: object) -> object: ... + def setdefault(self, k: _Never, default: object) -> object: ... # Mypy plugin hook for 'pop' expects that 'default' has a type variable type. - def pop(self, k: NoReturn, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] + def pop(self, k: _Never, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] def update(self: _T, __m: _T) -> None: ... - def __delitem__(self, k: NoReturn) -> None: ... - def items(self) -> ItemsView[str, object]: ... - def keys(self) -> KeysView[str]: ... - def values(self) -> ValuesView[object]: ... + def __delitem__(self, k: _Never) -> None: ... + def items(self) -> dict_items[str, object]: ... + def keys(self) -> dict_keys[str, object]: ... + def values(self) -> dict_values[str, object]: ... if sys.version_info >= (3, 9): def __or__(self: _typeshed.Self, __value: _typeshed.Self) -> _typeshed.Self: ... def __ior__(self: _typeshed.Self, __value: _typeshed.Self) -> _typeshed.Self: ... @@ -819,7 +821,13 @@ class ForwardRef: else: def __init__(self, arg: str, is_argument: bool = ...) -> None: ... - def _evaluate(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None) -> Any | None: ... + if sys.version_info >= (3, 9): + def _evaluate( + self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None, recursive_guard: frozenset[str] + ) -> Any | None: ... + else: + def _evaluate(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None) -> Any | None: ... + def __eq__(self, other: object) -> bool: ... if sys.version_info >= (3, 11): def __or__(self, other: Any) -> _SpecialForm: ... @@ -827,3 +835,5 @@ class ForwardRef: if sys.version_info >= (3, 10): def is_typeddict(tp: object) -> bool: ... + +def _type_repr(obj: object) -> str: ... diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index edc0d228e7a1..df2c1c431c65 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -2,11 +2,13 @@ import _typeshed import abc import collections import sys -from _typeshed import IdentityFunction +import typing +from _collections_abc import dict_items, dict_keys, dict_values +from _typeshed import IdentityFunction, Incomplete from collections.abc import Iterable from typing import ( # noqa: Y022,Y027,Y039 TYPE_CHECKING as TYPE_CHECKING, - Any, + Any as Any, AsyncContextManager as AsyncContextManager, AsyncGenerator as AsyncGenerator, AsyncIterable as AsyncIterable, @@ -20,22 +22,19 @@ from typing import ( # noqa: Y022,Y027,Y039 Counter as Counter, DefaultDict as DefaultDict, Deque as Deque, - ItemsView, - KeysView, Mapping, NewType as NewType, NoReturn as NoReturn, Sequence, Text as Text, Type as Type, - TypeVar, - ValuesView, _Alias, overload as overload, type_check_only, ) __all__ = [ + "Any", "ClassVar", "Concatenate", "Final", @@ -45,6 +44,7 @@ __all__ = [ "ParamSpecKwargs", "Self", "Type", + "TypeVar", "TypeVarTuple", "Unpack", "Awaitable", @@ -72,6 +72,7 @@ __all__ = [ "Literal", "NewType", "overload", + "override", "Protocol", "reveal_type", "runtime", @@ -91,9 +92,9 @@ __all__ = [ "get_type_hints", ] -_T = TypeVar("_T") -_F = TypeVar("_F", bound=Callable[..., Any]) -_TC = TypeVar("_TC", bound=Type[object]) +_T = typing.TypeVar("_T") +_F = typing.TypeVar("_F", bound=Callable[..., Any]) +_TC = typing.TypeVar("_TC", bound=Type[object]) # unfortunately we have to duplicate this class definition from typing.pyi or we break pytype class _SpecialForm: @@ -129,16 +130,16 @@ class _TypedDict(Mapping[str, object], metaclass=abc.ABCMeta): __optional_keys__: ClassVar[frozenset[str]] __total__: ClassVar[bool] def copy(self: _typeshed.Self) -> _typeshed.Self: ... - # Using NoReturn so that only calls using mypy plugin hook that specialize the signature + # Using Never so that only calls using mypy plugin hook that specialize the signature # can go through. - def setdefault(self, k: NoReturn, default: object) -> object: ... + def setdefault(self, k: Never, default: object) -> object: ... # Mypy plugin hook for 'pop' expects that 'default' has a type variable type. - def pop(self, k: NoReturn, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] + def pop(self, k: Never, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] def update(self: _T, __m: _T) -> None: ... - def items(self) -> ItemsView[str, object]: ... - def keys(self) -> KeysView[str]: ... - def values(self) -> ValuesView[object]: ... - def __delitem__(self, k: NoReturn) -> None: ... + def items(self) -> dict_items[str, object]: ... + def keys(self) -> dict_keys[str, object]: ... + def values(self) -> dict_values[str, object]: ... + def __delitem__(self, k: Never) -> None: ... if sys.version_info >= (3, 9): def __or__(self: _typeshed.Self, __value: _typeshed.Self) -> _typeshed.Self: ... def __ior__(self: _typeshed.Self, __value: _typeshed.Self) -> _typeshed.Self: ... @@ -169,7 +170,6 @@ class SupportsIndex(Protocol, metaclass=abc.ABCMeta): if sys.version_info >= (3, 10): from typing import ( Concatenate as Concatenate, - ParamSpec as ParamSpec, ParamSpecArgs as ParamSpecArgs, ParamSpecKwargs as ParamSpecKwargs, TypeAlias as TypeAlias, @@ -185,18 +185,6 @@ else: __origin__: ParamSpec def __init__(self, origin: ParamSpec) -> None: ... - class ParamSpec: - __name__: str - __bound__: type[Any] | None - __covariant__: bool - __contravariant__: bool - def __init__( - self, name: str, *, bound: None | type[Any] | str = ..., contravariant: bool = ..., covariant: bool = ... - ) -> None: ... - @property - def args(self) -> ParamSpecArgs: ... - @property - def kwargs(self) -> ParamSpecKwargs: ... Concatenate: _SpecialForm TypeAlias: _SpecialForm TypeGuard: _SpecialForm @@ -212,7 +200,6 @@ if sys.version_info >= (3, 11): NotRequired as NotRequired, Required as Required, Self as Self, - TypeVarTuple as TypeVarTuple, Unpack as Unpack, assert_never as assert_never, assert_type as assert_type, @@ -223,9 +210,9 @@ if sys.version_info >= (3, 11): ) else: Self: _SpecialForm - Never: _SpecialForm + Never: _SpecialForm = ... def reveal_type(__obj: _T) -> _T: ... - def assert_never(__arg: NoReturn) -> NoReturn: ... + def assert_never(__arg: Never) -> Never: ... def assert_type(__val: _T, __typ: Any) -> _T: ... def clear_overloads() -> None: ... def get_overloads(func: Callable[..., object]) -> Sequence[Callable[..., object]]: ... @@ -235,12 +222,6 @@ else: LiteralString: _SpecialForm Unpack: _SpecialForm - @final - class TypeVarTuple: - __name__: str - def __init__(self, name: str) -> None: ... - def __iter__(self) -> Any: ... # Unpack[Self] - def dataclass_transform( *, eq_default: bool = ..., @@ -270,3 +251,61 @@ else: def _asdict(self) -> collections.OrderedDict[str, Any]: ... def _replace(self: _typeshed.Self, **kwargs: Any) -> _typeshed.Self: ... + +# New things in 3.xx +# The `default` parameter was added to TypeVar, ParamSpec, and TypeVarTuple (PEP 696) +# The `infer_variance` parameter was added to TypeVar (PEP 695) +# typing_extensions.override (PEP 698) +@final +class TypeVar: + __name__: str + __bound__: Any | None + __constraints__: tuple[Any, ...] + __covariant__: bool + __contravariant__: bool + __default__: Any | None + def __init__( + self, + name: str, + *constraints: Any, + bound: Any | None = ..., + covariant: bool = ..., + contravariant: bool = ..., + default: Any | None = ..., + infer_variance: bool = ..., + ) -> None: ... + if sys.version_info >= (3, 10): + def __or__(self, right: Any) -> _SpecialForm: ... + def __ror__(self, left: Any) -> _SpecialForm: ... + if sys.version_info >= (3, 11): + def __typing_subst__(self, arg: Incomplete) -> Incomplete: ... + +@final +class ParamSpec: + __name__: str + __bound__: type[Any] | None + __covariant__: bool + __contravariant__: bool + __default__: type[Any] | None + def __init__( + self, + name: str, + *, + bound: None | type[Any] | str = ..., + contravariant: bool = ..., + covariant: bool = ..., + default: type[Any] | str | None = ..., + ) -> None: ... + @property + def args(self) -> ParamSpecArgs: ... + @property + def kwargs(self) -> ParamSpecKwargs: ... + +@final +class TypeVarTuple: + __name__: str + __default__: Any | None + def __init__(self, name: str, *, default: Any | None = ...) -> None: ... + def __iter__(self) -> Any: ... # Unpack[Self] + +def override(__arg: _F) -> _F: ... diff --git a/mypy/typeshed/stdlib/unittest/case.pyi b/mypy/typeshed/stdlib/unittest/case.pyi index 7db217077f1b..200f8dbaea23 100644 --- a/mypy/typeshed/stdlib/unittest/case.pyi +++ b/mypy/typeshed/stdlib/unittest/case.pyi @@ -65,7 +65,7 @@ else: ) -> bool | None: ... if sys.version_info >= (3, 8): - def addModuleCleanup(__function: Callable[_P, object], *args: _P.args, **kwargs: _P.kwargs) -> None: ... + def addModuleCleanup(__function: Callable[_P, Any], *args: _P.args, **kwargs: _P.kwargs) -> None: ... def doModuleCleanups() -> None: ... if sys.version_info >= (3, 11): @@ -150,13 +150,15 @@ class TestCase: **kwargs: Any, ) -> None: ... @overload - def assertRaises(self, expected_exception: type[_E] | tuple[type[_E], ...], msg: Any = ...) -> _AssertRaisesContext[_E]: ... + def assertRaises( + self, expected_exception: type[_E] | tuple[type[_E], ...], *, msg: Any = ... + ) -> _AssertRaisesContext[_E]: ... @overload def assertRaisesRegex( # type: ignore[misc] self, expected_exception: type[BaseException] | tuple[type[BaseException], ...], expected_regex: str | bytes | Pattern[str] | Pattern[bytes], - callable: Callable[..., object], + callable: Callable[..., Any], *args: Any, **kwargs: Any, ) -> None: ... @@ -165,24 +167,27 @@ class TestCase: self, expected_exception: type[_E] | tuple[type[_E], ...], expected_regex: str | bytes | Pattern[str] | Pattern[bytes], + *, msg: Any = ..., ) -> _AssertRaisesContext[_E]: ... @overload def assertWarns( # type: ignore[misc] self, expected_warning: type[Warning] | tuple[type[Warning], ...], - callable: Callable[_P, object], + callable: Callable[_P, Any], *args: _P.args, **kwargs: _P.kwargs, ) -> None: ... @overload - def assertWarns(self, expected_warning: type[Warning] | tuple[type[Warning], ...], msg: Any = ...) -> _AssertWarnsContext: ... + def assertWarns( + self, expected_warning: type[Warning] | tuple[type[Warning], ...], *, msg: Any = ... + ) -> _AssertWarnsContext: ... @overload def assertWarnsRegex( # type: ignore[misc] self, expected_warning: type[Warning] | tuple[type[Warning], ...], expected_regex: str | bytes | Pattern[str] | Pattern[bytes], - callable: Callable[_P, object], + callable: Callable[_P, Any], *args: _P.args, **kwargs: _P.kwargs, ) -> None: ... @@ -191,6 +196,7 @@ class TestCase: self, expected_warning: type[Warning] | tuple[type[Warning], ...], expected_regex: str | bytes | Pattern[str] | Pattern[bytes], + *, msg: Any = ..., ) -> _AssertWarnsContext: ... def assertLogs( @@ -267,9 +273,9 @@ class TestCase: def id(self) -> str: ... def shortDescription(self) -> str | None: ... if sys.version_info >= (3, 8): - def addCleanup(self, __function: Callable[_P, object], *args: _P.args, **kwargs: _P.kwargs) -> None: ... + def addCleanup(self, __function: Callable[_P, Any], *args: _P.args, **kwargs: _P.kwargs) -> None: ... else: - def addCleanup(self, function: Callable[_P, object], *args: _P.args, **kwargs: _P.kwargs) -> None: ... + def addCleanup(self, function: Callable[_P, Any], *args: _P.args, **kwargs: _P.kwargs) -> None: ... if sys.version_info >= (3, 11): def enterContext(self, cm: AbstractContextManager[_T]) -> _T: ... @@ -277,7 +283,7 @@ class TestCase: def doCleanups(self) -> None: ... if sys.version_info >= (3, 8): @classmethod - def addClassCleanup(cls, __function: Callable[_P, object], *args: _P.args, **kwargs: _P.kwargs) -> None: ... + def addClassCleanup(cls, __function: Callable[_P, Any], *args: _P.args, **kwargs: _P.kwargs) -> None: ... @classmethod def doClassCleanups(cls) -> None: ... @@ -310,9 +316,9 @@ class TestCase: class FunctionTestCase(TestCase): def __init__( self, - testFunc: Callable[[], object], - setUp: Callable[[], object] | None = ..., - tearDown: Callable[[], object] | None = ..., + testFunc: Callable[[], Any], + setUp: Callable[[], Any] | None = ..., + tearDown: Callable[[], Any] | None = ..., description: str | None = ..., ) -> None: ... def runTest(self) -> None: ... diff --git a/mypy/typeshed/stdlib/unittest/loader.pyi b/mypy/typeshed/stdlib/unittest/loader.pyi index 9ba04b084c7f..a1b902e0f6d6 100644 --- a/mypy/typeshed/stdlib/unittest/loader.pyi +++ b/mypy/typeshed/stdlib/unittest/loader.pyi @@ -23,6 +23,7 @@ class TestLoader: def loadTestsFromNames(self, names: Sequence[str], module: ModuleType | None = ...) -> unittest.suite.TestSuite: ... def getTestCaseNames(self, testCaseClass: type[unittest.case.TestCase]) -> Sequence[str]: ... def discover(self, start_dir: str, pattern: str = ..., top_level_dir: str | None = ...) -> unittest.suite.TestSuite: ... + def _match_path(self, path: str, full_path: str, pattern: str) -> bool: ... defaultTestLoader: TestLoader diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi index 4732994594f8..133380fce334 100644 --- a/mypy/typeshed/stdlib/unittest/mock.pyi +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -55,7 +55,6 @@ class _SentinelObject: def __init__(self, name: Any) -> None: ... class _Sentinel: - def __init__(self) -> None: ... def __getattr__(self, name: str) -> Any: ... sentinel: Any @@ -343,11 +342,8 @@ patch: _patcher class MagicMixin: def __init__(self, *args: Any, **kw: Any) -> None: ... -class NonCallableMagicMock(MagicMixin, NonCallableMock): - def mock_add_spec(self, spec: Any, spec_set: bool = ...) -> None: ... - -class MagicMock(MagicMixin, Mock): - def mock_add_spec(self, spec: Any, spec_set: bool = ...) -> None: ... +class NonCallableMagicMock(MagicMixin, NonCallableMock): ... +class MagicMock(MagicMixin, Mock): ... if sys.version_info >= (3, 8): class AsyncMockMixin(Base): diff --git a/mypy/typeshed/stdlib/unittest/runner.pyi b/mypy/typeshed/stdlib/unittest/runner.pyi index 1f1b89bc1bee..17514828898a 100644 --- a/mypy/typeshed/stdlib/unittest/runner.pyi +++ b/mypy/typeshed/stdlib/unittest/runner.pyi @@ -16,7 +16,6 @@ class TextTestResult(unittest.result.TestResult): stream: TextIO # undocumented def __init__(self, stream: TextIO, descriptions: bool, verbosity: int) -> None: ... def getDescription(self, test: unittest.case.TestCase) -> str: ... - def printErrors(self) -> None: ... def printErrorList(self, flavour: str, errors: Iterable[tuple[unittest.case.TestCase, str]]) -> None: ... class TextTestRunner: diff --git a/mypy/typeshed/stdlib/urllib/parse.pyi b/mypy/typeshed/stdlib/urllib/parse.pyi index 7e1ec903a15e..207a05e75a57 100644 --- a/mypy/typeshed/stdlib/urllib/parse.pyi +++ b/mypy/typeshed/stdlib/urllib/parse.pyi @@ -1,7 +1,6 @@ import sys from collections.abc import Callable, Mapping, Sequence from typing import Any, AnyStr, Generic, NamedTuple, overload -from typing_extensions import TypeAlias if sys.version_info >= (3, 9): from types import GenericAlias @@ -30,8 +29,6 @@ __all__ = [ "SplitResultBytes", ] -_Str: TypeAlias = bytes | str - uses_relative: list[str] uses_netloc: list[str] uses_params: list[str] @@ -135,16 +132,22 @@ def parse_qsl( separator: str = ..., ) -> list[tuple[AnyStr, AnyStr]]: ... @overload -def quote(string: str, safe: _Str = ..., encoding: str | None = ..., errors: str | None = ...) -> str: ... +def quote(string: str, safe: str | bytes = ..., encoding: str | None = ..., errors: str | None = ...) -> str: ... @overload -def quote(string: bytes, safe: _Str = ...) -> str: ... -def quote_from_bytes(bs: bytes, safe: _Str = ...) -> str: ... +def quote(string: bytes, safe: str | bytes = ...) -> str: ... +def quote_from_bytes(bs: bytes, safe: str | bytes = ...) -> str: ... @overload -def quote_plus(string: str, safe: _Str = ..., encoding: str | None = ..., errors: str | None = ...) -> str: ... +def quote_plus(string: str, safe: str | bytes = ..., encoding: str | None = ..., errors: str | None = ...) -> str: ... @overload -def quote_plus(string: bytes, safe: _Str = ...) -> str: ... -def unquote(string: str, encoding: str = ..., errors: str = ...) -> str: ... -def unquote_to_bytes(string: _Str) -> bytes: ... +def quote_plus(string: bytes, safe: str | bytes = ...) -> str: ... + +if sys.version_info >= (3, 9): + def unquote(string: str | bytes, encoding: str = ..., errors: str = ...) -> str: ... + +else: + def unquote(string: str, encoding: str = ..., errors: str = ...) -> str: ... + +def unquote_to_bytes(string: str | bytes) -> bytes: ... def unquote_plus(string: str, encoding: str = ..., errors: str = ...) -> str: ... @overload def urldefrag(url: str) -> DefragResult: ... @@ -153,10 +156,10 @@ def urldefrag(url: bytes | None) -> DefragResultBytes: ... def urlencode( query: Mapping[Any, Any] | Mapping[Any, Sequence[Any]] | Sequence[tuple[Any, Any]] | Sequence[tuple[Any, Sequence[Any]]], doseq: bool = ..., - safe: _Str = ..., + safe: str | bytes = ..., encoding: str = ..., errors: str = ..., - quote_via: Callable[[AnyStr, _Str, str, str], str] = ..., + quote_via: Callable[[AnyStr, str | bytes, str, str], str] = ..., ) -> str: ... def urljoin(base: AnyStr, url: AnyStr | None, allow_fragments: bool = ...) -> AnyStr: ... @overload diff --git a/mypy/typeshed/stdlib/urllib/request.pyi b/mypy/typeshed/stdlib/urllib/request.pyi index 88f4f5250e67..3cd5fc740fca 100644 --- a/mypy/typeshed/stdlib/urllib/request.pyi +++ b/mypy/typeshed/stdlib/urllib/request.pyi @@ -282,9 +282,6 @@ class CacheFTPHandler(FTPHandler): def setMaxConns(self, m: int) -> None: ... def check_cache(self) -> None: ... # undocumented def clear_cache(self) -> None: ... # undocumented - def connect_ftp( - self, user: str, passwd: str, host: str, port: int, dirs: str, timeout: float - ) -> ftpwrapper: ... # undocumented class UnknownHandler(BaseHandler): def unknown_open(self, req: Request) -> NoReturn: ... diff --git a/mypy/typeshed/stdlib/venv/__init__.pyi b/mypy/typeshed/stdlib/venv/__init__.pyi index 6afe328ac90d..2e34aed4c693 100644 --- a/mypy/typeshed/stdlib/venv/__init__.pyi +++ b/mypy/typeshed/stdlib/venv/__init__.pyi @@ -1,6 +1,6 @@ -from collections.abc import Sequence import sys from _typeshed import StrOrBytesPath +from collections.abc import Sequence from types import SimpleNamespace if sys.version_info >= (3, 9): diff --git a/mypy/typeshed/stdlib/wsgiref/simple_server.pyi b/mypy/typeshed/stdlib/wsgiref/simple_server.pyi index 1dc84e9fbebe..547f562cc1d4 100644 --- a/mypy/typeshed/stdlib/wsgiref/simple_server.pyi +++ b/mypy/typeshed/stdlib/wsgiref/simple_server.pyi @@ -12,7 +12,6 @@ software_version: str # undocumented class ServerHandler(SimpleHandler): # undocumented server_software: str - def close(self) -> None: ... class WSGIServer(HTTPServer): application: WSGIApplication | None @@ -25,7 +24,6 @@ class WSGIRequestHandler(BaseHTTPRequestHandler): server_version: str def get_environ(self) -> WSGIEnvironment: ... def get_stderr(self) -> ErrorStream: ... - def handle(self) -> None: ... def demo_app(environ: WSGIEnvironment, start_response: StartResponse) -> list[bytes]: ... diff --git a/mypy/typeshed/stdlib/xdrlib.pyi b/mypy/typeshed/stdlib/xdrlib.pyi index e0b8c6a54b00..78f3ecec8d78 100644 --- a/mypy/typeshed/stdlib/xdrlib.pyi +++ b/mypy/typeshed/stdlib/xdrlib.pyi @@ -12,7 +12,6 @@ class Error(Exception): class ConversionError(Error): ... class Packer: - def __init__(self) -> None: ... def reset(self) -> None: ... def get_buffer(self) -> bytes: ... def get_buf(self) -> bytes: ... diff --git a/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi b/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi index 58914e8fabf1..3ca885dbbaa0 100644 --- a/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi +++ b/mypy/typeshed/stdlib/xml/dom/expatbuilder.pyi @@ -1,3 +1,4 @@ +from _typeshed import Incomplete from typing import Any, NoReturn from xml.dom.minidom import Document, DOMImplementation, Node, TypeInfo from xml.dom.xmlbuilder import DOMBuilderFilter, Options @@ -12,8 +13,8 @@ FILTER_INTERRUPT = DOMBuilderFilter.FILTER_INTERRUPT theDOMImplementation: DOMImplementation | None class ElementInfo: - tagName: Any - def __init__(self, tagName, model: Any | None = ...) -> None: ... + tagName: Incomplete + def __init__(self, tagName, model: Incomplete | None = ...) -> None: ... def getAttributeType(self, aname) -> TypeInfo: ... def getAttributeTypeNS(self, namespaceURI, localName) -> TypeInfo: ... def isElementContent(self) -> bool: ... @@ -23,7 +24,7 @@ class ElementInfo: class ExpatBuilder: document: Document # Created in self.reset() - curNode: Any # Created in self.reset() + curNode: Incomplete # Created in self.reset() def __init__(self, options: Options | None = ...) -> None: ... def createParser(self): ... def getParser(self): ... @@ -67,9 +68,9 @@ class Skipper(FilterCrutch): def end_element_handler(self, *args: Any) -> None: ... class FragmentBuilder(ExpatBuilder): - fragment: Any | None - originalDocument: Any - context: Any + fragment: Incomplete | None + originalDocument: Incomplete + context: Incomplete def __init__(self, context, options: Options | None = ...) -> None: ... class Namespaces: diff --git a/mypy/typeshed/stdlib/xml/dom/minidom.pyi b/mypy/typeshed/stdlib/xml/dom/minidom.pyi index 7645bd79e9c1..04086fdc81b1 100644 --- a/mypy/typeshed/stdlib/xml/dom/minidom.pyi +++ b/mypy/typeshed/stdlib/xml/dom/minidom.pyi @@ -1,7 +1,6 @@ import sys import xml.dom -from _typeshed import Self, SupportsRead -from typing import Any +from _typeshed import Incomplete, Self, SupportsRead, SupportsWrite from typing_extensions import Literal from xml.dom.xmlbuilder import DocumentLS, DOMImplementationLS from xml.sax.xmlreader import XMLReader @@ -12,11 +11,11 @@ def getDOMImplementation(features=...) -> DOMImplementation | None: ... class Node(xml.dom.Node): namespaceURI: str | None - parentNode: Any - ownerDocument: Any - nextSibling: Any - previousSibling: Any - prefix: Any + parentNode: Incomplete + ownerDocument: Incomplete + nextSibling: Incomplete + previousSibling: Incomplete + prefix: Incomplete @property def firstChild(self) -> Node | None: ... @property @@ -25,11 +24,11 @@ class Node(xml.dom.Node): def localName(self) -> str | None: ... def __bool__(self) -> Literal[True]: ... if sys.version_info >= (3, 9): - def toxml(self, encoding: Any | None = ..., standalone: Any | None = ...): ... - def toprettyxml(self, indent: str = ..., newl: str = ..., encoding: Any | None = ..., standalone: Any | None = ...): ... + def toxml(self, encoding: str | None = ..., standalone: bool | None = ...): ... + def toprettyxml(self, indent: str = ..., newl: str = ..., encoding: str | None = ..., standalone: bool | None = ...): ... else: - def toxml(self, encoding: Any | None = ...): ... - def toprettyxml(self, indent: str = ..., newl: str = ..., encoding: Any | None = ...): ... + def toxml(self, encoding: str | None = ...): ... + def toprettyxml(self, indent: str = ..., newl: str = ..., encoding: str | None = ...): ... def hasChildNodes(self) -> bool: ... def insertBefore(self, newChild, refChild): ... @@ -43,40 +42,40 @@ class Node(xml.dom.Node): def getInterface(self, feature): ... def getUserData(self, key): ... def setUserData(self, key, data, handler): ... - childNodes: Any + childNodes: Incomplete def unlink(self) -> None: ... def __enter__(self: Self) -> Self: ... def __exit__(self, et, ev, tb) -> None: ... class DocumentFragment(Node): - nodeType: Any + nodeType: int nodeName: str - nodeValue: Any - attributes: Any - parentNode: Any - childNodes: Any + nodeValue: Incomplete + attributes: Incomplete + parentNode: Incomplete + childNodes: Incomplete def __init__(self) -> None: ... class Attr(Node): name: str - nodeType: Any - attributes: Any + nodeType: int + attributes: Incomplete specified: bool - ownerElement: Any + ownerElement: Incomplete namespaceURI: str | None - childNodes: Any - nodeName: Any + childNodes: Incomplete + nodeName: Incomplete nodeValue: str value: str - prefix: Any + prefix: Incomplete def __init__( - self, qName: str, namespaceURI: str | None = ..., localName: Any | None = ..., prefix: Any | None = ... + self, qName: str, namespaceURI: str | None = ..., localName: str | None = ..., prefix: Incomplete | None = ... ) -> None: ... def unlink(self) -> None: ... @property def isId(self) -> bool: ... @property - def schemaType(self) -> Any: ... + def schemaType(self): ... class NamedNodeMap: def __init__(self, attrs, attrsNS, ownerElement) -> None: ... @@ -87,45 +86,45 @@ class NamedNodeMap: def keys(self): ... def keysNS(self): ... def values(self): ... - def get(self, name, value: Any | None = ...): ... + def get(self, name: str, value: Incomplete | None = ...): ... def __len__(self) -> int: ... def __eq__(self, other: object) -> bool: ... - def __ge__(self, other: Any) -> bool: ... - def __gt__(self, other: Any) -> bool: ... - def __le__(self, other: Any) -> bool: ... - def __lt__(self, other: Any) -> bool: ... - def __getitem__(self, attname_or_tuple): ... - def __setitem__(self, attname, value) -> None: ... - def getNamedItem(self, name): ... - def getNamedItemNS(self, namespaceURI: str, localName): ... - def removeNamedItem(self, name): ... - def removeNamedItemNS(self, namespaceURI: str, localName): ... - def setNamedItem(self, node): ... - def setNamedItemNS(self, node): ... - def __delitem__(self, attname_or_tuple) -> None: ... + def __ge__(self, other: NamedNodeMap) -> bool: ... + def __gt__(self, other: NamedNodeMap) -> bool: ... + def __le__(self, other: NamedNodeMap) -> bool: ... + def __lt__(self, other: NamedNodeMap) -> bool: ... + def __getitem__(self, attname_or_tuple: tuple[str, str | None] | str): ... + def __setitem__(self, attname: str, value: Attr | str) -> None: ... + def getNamedItem(self, name: str) -> Attr | None: ... + def getNamedItemNS(self, namespaceURI: str, localName: str | None) -> Attr | None: ... + def removeNamedItem(self, name: str) -> Attr: ... + def removeNamedItemNS(self, namespaceURI: str, localName: str | None): ... + def setNamedItem(self, node: Attr) -> Attr: ... + def setNamedItemNS(self, node: Attr) -> Attr: ... + def __delitem__(self, attname_or_tuple: tuple[str, str | None] | str) -> None: ... @property def length(self) -> int: ... AttributeList = NamedNodeMap class TypeInfo: - namespace: Any - name: Any - def __init__(self, namespace, name) -> None: ... + namespace: Incomplete | None + name: str + def __init__(self, namespace: Incomplete | None, name: str) -> None: ... class Element(Node): - nodeType: Any - nodeValue: Any - schemaType: Any - parentNode: Any + nodeType: int + nodeValue: Incomplete + schemaType: Incomplete + parentNode: Incomplete tagName: str nodeName: str - prefix: Any + prefix: Incomplete namespaceURI: str | None - childNodes: Any - nextSibling: Any + childNodes: Incomplete + nextSibling: Incomplete def __init__( - self, tagName, namespaceURI: str | None = ..., prefix: Any | None = ..., localName: Any | None = ... + self, tagName, namespaceURI: str | None = ..., prefix: Incomplete | None = ..., localName: Incomplete | None = ... ) -> None: ... def unlink(self) -> None: ... def getAttribute(self, attname: str) -> str: ... @@ -135,16 +134,16 @@ class Element(Node): def getAttributeNode(self, attrname: str): ... def getAttributeNodeNS(self, namespaceURI: str, localName): ... def setAttributeNode(self, attr): ... - setAttributeNodeNS: Any + setAttributeNodeNS: Incomplete def removeAttribute(self, name: str) -> None: ... def removeAttributeNS(self, namespaceURI: str, localName) -> None: ... def removeAttributeNode(self, node): ... - removeAttributeNodeNS: Any + removeAttributeNodeNS: Incomplete def hasAttribute(self, name: str) -> bool: ... def hasAttributeNS(self, namespaceURI: str, localName) -> bool: ... def getElementsByTagName(self, name: str): ... def getElementsByTagNameNS(self, namespaceURI: str, localName): ... - def writexml(self, writer, indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... + def writexml(self, writer: SupportsWrite[str], indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... def hasAttributes(self) -> bool: ... def setIdAttribute(self, name) -> None: ... def setIdAttributeNS(self, namespaceURI: str, localName) -> None: ... @@ -153,10 +152,10 @@ class Element(Node): def attributes(self) -> NamedNodeMap: ... class Childless: - attributes: Any - childNodes: Any - firstChild: Any - lastChild: Any + attributes: Incomplete + childNodes: Incomplete + firstChild: Incomplete + lastChild: Incomplete def appendChild(self, node) -> None: ... def hasChildNodes(self) -> bool: ... def insertBefore(self, newChild, refChild) -> None: ... @@ -165,21 +164,21 @@ class Childless: def replaceChild(self, newChild, oldChild) -> None: ... class ProcessingInstruction(Childless, Node): - nodeType: Any - target: Any - data: Any + nodeType: int + target: Incomplete + data: Incomplete def __init__(self, target, data) -> None: ... - nodeValue: Any - nodeName: Any - def writexml(self, writer, indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... + nodeValue: Incomplete + nodeName: Incomplete + def writexml(self, writer: SupportsWrite[str], indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... class CharacterData(Childless, Node): - ownerDocument: Any - previousSibling: Any + ownerDocument: Incomplete + previousSibling: Incomplete def __init__(self) -> None: ... def __len__(self) -> int: ... data: str - nodeValue: Any + nodeValue: Incomplete def substringData(self, offset: int, count: int) -> str: ... def appendData(self, arg: str) -> None: ... def insertData(self, offset: int, arg: str) -> None: ... @@ -189,12 +188,12 @@ class CharacterData(Childless, Node): def length(self) -> int: ... class Text(CharacterData): - nodeType: Any + nodeType: int nodeName: str - attributes: Any - data: Any + attributes: Incomplete + data: Incomplete def splitText(self, offset): ... - def writexml(self, writer, indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... + def writexml(self, writer: SupportsWrite[str], indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... def replaceWholeText(self, content): ... @property def isWhitespaceInElementContent(self) -> bool: ... @@ -202,15 +201,15 @@ class Text(CharacterData): def wholeText(self) -> str: ... class Comment(CharacterData): - nodeType: Any + nodeType: int nodeName: str def __init__(self, data) -> None: ... - def writexml(self, writer, indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... + def writexml(self, writer: SupportsWrite[str], indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... class CDATASection(Text): - nodeType: Any + nodeType: int nodeName: str - def writexml(self, writer, indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... + def writexml(self, writer: SupportsWrite[str], indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... class ReadOnlySequentialNamedNodeMap: def __init__(self, seq=...) -> None: ... @@ -227,31 +226,31 @@ class ReadOnlySequentialNamedNodeMap: def length(self) -> int: ... class Identified: - publicId: Any - systemId: Any + publicId: Incomplete + systemId: Incomplete class DocumentType(Identified, Childless, Node): - nodeType: Any - nodeValue: Any - name: Any - internalSubset: Any - entities: Any - notations: Any - nodeName: Any + nodeType: int + nodeValue: Incomplete + name: Incomplete + internalSubset: Incomplete + entities: Incomplete + notations: Incomplete + nodeName: Incomplete def __init__(self, qualifiedName: str) -> None: ... def cloneNode(self, deep): ... - def writexml(self, writer, indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... + def writexml(self, writer: SupportsWrite[str], indent: str = ..., addindent: str = ..., newl: str = ...) -> None: ... class Entity(Identified, Node): - attributes: Any - nodeType: Any - nodeValue: Any - actualEncoding: Any - encoding: Any - version: Any - nodeName: Any - notationName: Any - childNodes: Any + attributes: Incomplete + nodeType: int + nodeValue: Incomplete + actualEncoding: Incomplete + encoding: Incomplete + version: Incomplete + nodeName: Incomplete + notationName: Incomplete + childNodes: Incomplete def __init__(self, name, publicId, systemId, notation) -> None: ... def appendChild(self, newChild) -> None: ... def insertBefore(self, newChild, refChild) -> None: ... @@ -259,19 +258,19 @@ class Entity(Identified, Node): def replaceChild(self, newChild, oldChild) -> None: ... class Notation(Identified, Childless, Node): - nodeType: Any - nodeValue: Any - nodeName: Any + nodeType: int + nodeValue: Incomplete + nodeName: Incomplete def __init__(self, name, publicId, systemId) -> None: ... class DOMImplementation(DOMImplementationLS): - def hasFeature(self, feature, version) -> bool: ... - def createDocument(self, namespaceURI: str | None, qualifiedName: str | None, doctype): ... - def createDocumentType(self, qualifiedName: str | None, publicId, systemId): ... - def getInterface(self, feature): ... + def hasFeature(self, feature: str, version: str | None) -> bool: ... + def createDocument(self, namespaceURI: str | None, qualifiedName: str | None, doctype: DocumentType | None) -> Document: ... + def createDocumentType(self, qualifiedName: str | None, publicId: str, systemId: str) -> DocumentType: ... + def getInterface(self: Self, feature: str) -> Self | None: ... class ElementInfo: - tagName: Any + tagName: Incomplete def __init__(self, name) -> None: ... def getAttributeType(self, aname): ... def getAttributeTypeNS(self, namespaceURI: str, localName): ... @@ -281,34 +280,34 @@ class ElementInfo: def isIdNS(self, namespaceURI: str, localName): ... class Document(Node, DocumentLS): - implementation: Any - nodeType: Any + implementation: Incomplete + nodeType: int nodeName: str - nodeValue: Any - attributes: Any - parentNode: Any - previousSibling: Any - nextSibling: Any - actualEncoding: Any - encoding: Any - standalone: Any - version: Any + nodeValue: Incomplete + attributes: Incomplete + parentNode: Incomplete + previousSibling: Incomplete + nextSibling: Incomplete + actualEncoding: Incomplete + encoding: str | None + standalone: bool | None + version: Incomplete strictErrorChecking: bool - errorHandler: Any - documentURI: Any - doctype: Any - childNodes: Any + errorHandler: Incomplete + documentURI: Incomplete + doctype: DocumentType | None + childNodes: Incomplete def __init__(self) -> None: ... def appendChild(self, node): ... - documentElement: Any + documentElement: Incomplete def removeChild(self, oldChild): ... def unlink(self) -> None: ... def cloneNode(self, deep): ... - def createDocumentFragment(self): ... - def createElement(self, tagName: str): ... - def createTextNode(self, data): ... - def createCDATASection(self, data): ... - def createComment(self, data): ... + def createDocumentFragment(self) -> DocumentFragment: ... + def createElement(self, tagName: str) -> Element: ... + def createTextNode(self, data: str) -> Text: ... + def createCDATASection(self, data: str) -> CDATASection: ... + def createComment(self, data: str) -> Comment: ... def createProcessingInstruction(self, target, data): ... def createAttribute(self, qName) -> Attr: ... def createElementNS(self, namespaceURI: str, qualifiedName: str): ... @@ -316,21 +315,26 @@ class Document(Node, DocumentLS): def getElementById(self, id): ... def getElementsByTagName(self, name: str): ... def getElementsByTagNameNS(self, namespaceURI: str, localName): ... - def isSupported(self, feature, version): ... + def isSupported(self, feature: str, version: str | None) -> bool: ... def importNode(self, node, deep): ... if sys.version_info >= (3, 9): def writexml( self, - writer, + writer: SupportsWrite[str], indent: str = ..., addindent: str = ..., newl: str = ..., - encoding: Any | None = ..., - standalone: Any | None = ..., + encoding: str | None = ..., + standalone: bool | None = ..., ) -> None: ... else: def writexml( - self, writer, indent: str = ..., addindent: str = ..., newl: str = ..., encoding: Any | None = ... + self, + writer: SupportsWrite[str], + indent: str = ..., + addindent: str = ..., + newl: str = ..., + encoding: Incomplete | None = ..., ) -> None: ... def renameNode(self, n, namespaceURI: str, name): ... diff --git a/mypy/typeshed/stdlib/xml/dom/pulldom.pyi b/mypy/typeshed/stdlib/xml/dom/pulldom.pyi index 07f220ddd347..b4c03a1dd590 100644 --- a/mypy/typeshed/stdlib/xml/dom/pulldom.pyi +++ b/mypy/typeshed/stdlib/xml/dom/pulldom.pyi @@ -1,7 +1,6 @@ import sys -from _typeshed import SupportsRead +from _typeshed import Incomplete, SupportsRead from collections.abc import Sequence -from typing import Any from typing_extensions import Literal, TypeAlias from xml.dom.minidom import Document, DOMImplementation, Element, Text from xml.sax.handler import ContentHandler @@ -36,10 +35,10 @@ _Event: TypeAlias = tuple[ class PullDOM(ContentHandler): document: Document | None documentFactory: _DocumentFactory - firstEvent: Any - lastEvent: Any - elementStack: Sequence[Any] - pending_events: Sequence[Any] + firstEvent: Incomplete + lastEvent: Incomplete + elementStack: Sequence[Incomplete] + pending_events: Sequence[Incomplete] def __init__(self, documentFactory: _DocumentFactory = ...) -> None: ... def pop(self) -> Element: ... def setDocumentLocator(self, locator) -> None: ... @@ -68,7 +67,7 @@ class DOMEventStream: parser: XMLReader bufsize: int def __init__(self, stream: SupportsRead[bytes] | SupportsRead[str], parser: XMLReader, bufsize: int) -> None: ... - pulldom: Any + pulldom: Incomplete if sys.version_info < (3, 11): def __getitem__(self, pos): ... diff --git a/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi b/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi index f6afd8aa2786..341d717e043b 100644 --- a/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi +++ b/mypy/typeshed/stdlib/xml/dom/xmlbuilder.pyi @@ -1,3 +1,4 @@ +from _typeshed import Incomplete from typing import Any, NoReturn from typing_extensions import Literal, TypeAlias from urllib.request import OpenerDirector @@ -11,20 +12,20 @@ __all__ = ["DOMBuilder", "DOMEntityResolver", "DOMInputSource"] # The same as `_DOMBuilderErrorHandlerType`? # Maybe `xml.sax.handler.ErrorHandler`? # - Return type of DOMBuilder.getFeature(). -# We could get rid of the `Any` if we knew more +# We could get rid of the `Incomplete` if we knew more # about `Options.errorHandler`. # ALIASES REPRESENTING MORE UNKNOWN TYPES: # probably the same as `Options.errorHandler`? # Maybe `xml.sax.handler.ErrorHandler`? -_DOMBuilderErrorHandlerType: TypeAlias = Any | None +_DOMBuilderErrorHandlerType: TypeAlias = Incomplete | None # probably some kind of IO... -_DOMInputSourceCharacterStreamType: TypeAlias = Any | None +_DOMInputSourceCharacterStreamType: TypeAlias = Incomplete | None # probably a string?? -_DOMInputSourceStringDataType: TypeAlias = Any | None +_DOMInputSourceStringDataType: TypeAlias = Incomplete | None # probably a string?? -_DOMInputSourceEncodingType: TypeAlias = Any | None +_DOMInputSourceEncodingType: TypeAlias = Incomplete | None class Options: namespaces: int @@ -55,12 +56,11 @@ class DOMBuilder: ACTION_APPEND_AS_CHILDREN: Literal[2] ACTION_INSERT_AFTER: Literal[3] ACTION_INSERT_BEFORE: Literal[4] - def __init__(self) -> None: ... def setFeature(self, name: str, state: int) -> None: ... def supportsFeature(self, name: str) -> bool: ... def canSetFeature(self, name: str, state: int) -> bool: ... # getFeature could return any attribute from an instance of `Options` - def getFeature(self, name: str) -> Any: ... + def getFeature(self, name: str) -> Incomplete: ... def parseURI(self, uri: str) -> ExpatBuilder | ExpatBuilderNS: ... def parse(self, input: DOMInputSource) -> ExpatBuilder | ExpatBuilderNS: ... # `input` and `cnode` argtypes for `parseWithContext` are unknowable diff --git a/mypy/typeshed/stdlib/xml/sax/handler.pyi b/mypy/typeshed/stdlib/xml/sax/handler.pyi index abf124f836cd..63b725bd6da6 100644 --- a/mypy/typeshed/stdlib/xml/sax/handler.pyi +++ b/mypy/typeshed/stdlib/xml/sax/handler.pyi @@ -1,14 +1,14 @@ import sys +from typing import NoReturn version: str class ErrorHandler: - def error(self, exception): ... - def fatalError(self, exception): ... - def warning(self, exception): ... + def error(self, exception: BaseException) -> NoReturn: ... + def fatalError(self, exception: BaseException) -> NoReturn: ... + def warning(self, exception: BaseException) -> None: ... class ContentHandler: - def __init__(self) -> None: ... def setDocumentLocator(self, locator): ... def startDocument(self): ... def endDocument(self): ... diff --git a/mypy/typeshed/stdlib/xml/sax/xmlreader.pyi b/mypy/typeshed/stdlib/xml/sax/xmlreader.pyi index d7d4db5b0a16..517c17072b87 100644 --- a/mypy/typeshed/stdlib/xml/sax/xmlreader.pyi +++ b/mypy/typeshed/stdlib/xml/sax/xmlreader.pyi @@ -1,7 +1,6 @@ from collections.abc import Mapping class XMLReader: - def __init__(self) -> None: ... def parse(self, source): ... def getContentHandler(self): ... def setContentHandler(self, handler): ... diff --git a/mypy/typeshed/stdlib/xmlrpc/client.pyi b/mypy/typeshed/stdlib/xmlrpc/client.pyi index 7c0ba5c62fd7..150291009f54 100644 --- a/mypy/typeshed/stdlib/xmlrpc/client.pyi +++ b/mypy/typeshed/stdlib/xmlrpc/client.pyi @@ -203,7 +203,6 @@ class GzipDecodedResponse(gzip.GzipFile): # undocumented io: BytesIO def __init__(self, response: SupportsRead[bytes]) -> None: ... - def close(self) -> None: ... class _Method: # undocumented diff --git a/mypy/typeshed/stdlib/xmlrpc/server.pyi b/mypy/typeshed/stdlib/xmlrpc/server.pyi index e4fc300343bf..c11d8d8e7a14 100644 --- a/mypy/typeshed/stdlib/xmlrpc/server.pyi +++ b/mypy/typeshed/stdlib/xmlrpc/server.pyi @@ -72,11 +72,9 @@ class SimpleXMLRPCRequestHandler(http.server.BaseHTTPRequestHandler): def do_POST(self) -> None: ... def decode_request_content(self, data: bytes) -> bytes | None: ... def report_404(self) -> None: ... - def log_request(self, code: int | str = ..., size: int | str = ...) -> None: ... class SimpleXMLRPCServer(socketserver.TCPServer, SimpleXMLRPCDispatcher): - allow_reuse_address: bool _send_traceback_handler: bool def __init__( self, @@ -92,8 +90,6 @@ class SimpleXMLRPCServer(socketserver.TCPServer, SimpleXMLRPCDispatcher): class MultiPathXMLRPCServer(SimpleXMLRPCServer): # undocumented dispatchers: dict[str, SimpleXMLRPCDispatcher] - allow_none: bool - encoding: str def __init__( self, addr: tuple[str, int], @@ -106,12 +102,6 @@ class MultiPathXMLRPCServer(SimpleXMLRPCServer): # undocumented ) -> None: ... def add_dispatcher(self, path: str, dispatcher: SimpleXMLRPCDispatcher) -> SimpleXMLRPCDispatcher: ... def get_dispatcher(self, path: str) -> SimpleXMLRPCDispatcher: ... - def _marshaled_dispatch( - self, - data: str, - dispatch_method: Callable[[str | None, tuple[_Marshallable, ...]], Fault | tuple[_Marshallable, ...]] | None = ..., - path: Any | None = ..., - ) -> str: ... class CGIXMLRPCRequestHandler(SimpleXMLRPCDispatcher): def __init__(self, allow_none: bool = ..., encoding: str | None = ..., use_builtin_types: bool = ...) -> None: ... @@ -137,7 +127,6 @@ class XMLRPCDocGenerator: # undocumented server_name: str server_documentation: str server_title: str - def __init__(self) -> None: ... def set_server_title(self, server_title: str) -> None: ... def set_server_name(self, server_name: str) -> None: ... def set_server_documentation(self, server_documentation: str) -> None: ... diff --git a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi index 1a0760862733..7f22c07b32c0 100644 --- a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi +++ b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi @@ -18,7 +18,7 @@ class ZoneInfo(tzinfo): @classmethod def from_file(cls: type[Self], __fobj: _IOBytes, key: str | None = ...) -> Self: ... @classmethod - def clear_cache(cls, *, only_keys: Iterable[str] = ...) -> None: ... + def clear_cache(cls, *, only_keys: Iterable[str] | None = ...) -> None: ... # Note: Both here and in clear_cache, the types allow the use of `str` where # a sequence of strings is required. This should be remedied if a solution diff --git a/mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi b/mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi index edefcc318176..47547942b2e7 100644 --- a/mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi +++ b/mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi @@ -1,8 +1,10 @@ import abc import sys +from _collections_abc import dict_items, dict_keys, dict_values from _typeshed import IdentityFunction, Self -from collections.abc import ItemsView, KeysView, Mapping, ValuesView +from collections.abc import Mapping from typing import Any, ClassVar, Generic, TypeVar, overload, type_check_only +from typing_extensions import Never _T = TypeVar("_T") _U = TypeVar("_U") @@ -15,16 +17,16 @@ class _TypedDict(Mapping[str, object], metaclass=abc.ABCMeta): # Unlike typing(_extensions).TypedDict, # subclasses of mypy_extensions.TypedDict do NOT have the __required_keys__ and __optional_keys__ ClassVars def copy(self: Self) -> Self: ... - # Using NoReturn so that only calls using mypy plugin hook that specialize the signature + # Using Never so that only calls using mypy plugin hook that specialize the signature # can go through. - def setdefault(self, k: NoReturn, default: object) -> object: ... + def setdefault(self, k: Never, default: object) -> object: ... # Mypy plugin hook for 'pop' expects that 'default' has a type variable type. - def pop(self, k: NoReturn, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] + def pop(self, k: Never, default: _T = ...) -> object: ... # pyright: ignore[reportInvalidTypeVarUse] def update(self: Self, __m: Self) -> None: ... - def items(self) -> ItemsView[str, object]: ... - def keys(self) -> KeysView[str]: ... - def values(self) -> ValuesView[object]: ... - def __delitem__(self, k: NoReturn) -> None: ... + def items(self) -> dict_items[str, object]: ... + def keys(self) -> dict_keys[str, object]: ... + def values(self) -> dict_values[str, object]: ... + def __delitem__(self, k: Never) -> None: ... if sys.version_info >= (3, 9): def __or__(self: Self, __other: Self) -> Self: ... def __ior__(self: Self, __other: Self) -> Self: ... diff --git a/mypy/typevartuples.py b/mypy/typevartuples.py index a63ebf3bfe08..e93f99d8a825 100644 --- a/mypy/typevartuples.py +++ b/mypy/typevartuples.py @@ -44,6 +44,105 @@ def split_with_instance( ) +def split_with_mapped_and_template( + mapped: Instance, template: Instance +) -> tuple[ + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], +] | None: + split_result = fully_split_with_mapped_and_template(mapped, template) + if split_result is None: + return None + + ( + mapped_prefix, + mapped_middle_prefix, + mapped_middle_middle, + mapped_middle_suffix, + mapped_suffix, + template_prefix, + template_middle_prefix, + template_middle_middle, + template_middle_suffix, + template_suffix, + ) = split_result + + return ( + mapped_prefix + mapped_middle_prefix, + mapped_middle_middle, + mapped_middle_suffix + mapped_suffix, + template_prefix + template_middle_prefix, + template_middle_middle, + template_middle_suffix + template_suffix, + ) + + +def fully_split_with_mapped_and_template( + mapped: Instance, template: Instance +) -> tuple[ + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], + tuple[Type, ...], +] | None: + mapped_prefix, mapped_middle, mapped_suffix = split_with_instance(mapped) + template_prefix, template_middle, template_suffix = split_with_instance(template) + + unpack_prefix = find_unpack_in_list(template_middle) + if unpack_prefix is None: + return ( + mapped_prefix, + (), + mapped_middle, + (), + mapped_suffix, + template_prefix, + (), + template_middle, + (), + template_suffix, + ) + + unpack_suffix = len(template_middle) - unpack_prefix - 1 + # mapped_middle is too short to do the unpack + if unpack_prefix + unpack_suffix > len(mapped_middle): + return None + + ( + mapped_middle_prefix, + mapped_middle_middle, + mapped_middle_suffix, + ) = split_with_prefix_and_suffix(mapped_middle, unpack_prefix, unpack_suffix) + ( + template_middle_prefix, + template_middle_middle, + template_middle_suffix, + ) = split_with_prefix_and_suffix(template_middle, unpack_prefix, unpack_suffix) + + return ( + mapped_prefix, + mapped_middle_prefix, + mapped_middle_middle, + mapped_middle_suffix, + mapped_suffix, + template_prefix, + template_middle_prefix, + template_middle_middle, + template_middle_suffix, + template_suffix, + ) + + def extract_unpack(types: Sequence[Type]) -> ProperType | None: """Given a list of types, extracts either a single type from an unpack, or returns None.""" if len(types) == 1: diff --git a/mypy/util.py b/mypy/util.py index 686a71c4331b..e836aefb3c7e 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -25,11 +25,14 @@ T = TypeVar("T") -with importlib_resources.path( - "mypy", # mypy-c doesn't support __package__ - "py.typed", # a marker file for type information, we assume typeshed to live in the same dir -) as _resource: - TYPESHED_DIR: Final = str(_resource.parent / "typeshed") +if sys.version_info >= (3, 9): + TYPESHED_DIR: Final = str(importlib_resources.files("mypy") / "typeshed") +else: + with importlib_resources.path( + "mypy", # mypy-c doesn't support __package__ + "py.typed", # a marker file for type information, we assume typeshed to live in the same dir + ) as _resource: + TYPESHED_DIR = str(_resource.parent / "typeshed") ENCODING_RE: Final = re.compile(rb"([ \t\v]*#.*(\r\n?|\n))??[ \t\v]*#.*coding[:=][ \t]*([-\w.]+)") @@ -516,24 +519,29 @@ def parse_gray_color(cup: bytes) -> str: return gray +def should_force_color() -> bool: + return bool(int(os.getenv("MYPY_FORCE_COLOR", os.getenv("FORCE_COLOR", "0")))) + + class FancyFormatter: """Apply color and bold font to terminal output. This currently only works on Linux and Mac. """ - def __init__(self, f_out: IO[str], f_err: IO[str], show_error_codes: bool) -> None: - self.show_error_codes = show_error_codes + def __init__(self, f_out: IO[str], f_err: IO[str], hide_error_codes: bool) -> None: + self.hide_error_codes = hide_error_codes # Check if we are in a human-facing terminal on a supported platform. - if sys.platform not in ("linux", "darwin", "win32"): + if sys.platform not in ("linux", "darwin", "win32", "emscripten"): self.dummy_term = True return - force_color = int(os.getenv("MYPY_FORCE_COLOR", "0")) - if not force_color and (not f_out.isatty() or not f_err.isatty()): + if not should_force_color() and (not f_out.isatty() or not f_err.isatty()): self.dummy_term = True return if sys.platform == "win32": self.dummy_term = not self.initialize_win_colors() + elif sys.platform == "emscripten": + self.dummy_term = not self.initialize_vt100_colors() else: self.dummy_term = not self.initialize_unix_colors() if not self.dummy_term: @@ -545,6 +553,20 @@ def __init__(self, f_out: IO[str], f_err: IO[str], show_error_codes: bool) -> No "none": "", } + def initialize_vt100_colors(self) -> bool: + """Return True if initialization was successful and we can use colors, False otherwise""" + # Windows and Emscripten can both use ANSI/VT100 escape sequences for color + assert sys.platform in ("win32", "emscripten") + self.BOLD = "\033[1m" + self.UNDER = "\033[4m" + self.BLUE = "\033[94m" + self.GREEN = "\033[92m" + self.RED = "\033[91m" + self.YELLOW = "\033[93m" + self.NORMAL = "\033[0m" + self.DIM = "\033[2m" + return True + def initialize_win_colors(self) -> bool: """Return True if initialization was successful and we can use colors, False otherwise""" # Windows ANSI escape sequences are only supported on Threshold 2 and above. @@ -571,14 +593,7 @@ def initialize_win_colors(self) -> bool: | ENABLE_WRAP_AT_EOL_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING, ) - self.BOLD = "\033[1m" - self.UNDER = "\033[4m" - self.BLUE = "\033[94m" - self.GREEN = "\033[92m" - self.RED = "\033[91m" - self.YELLOW = "\033[93m" - self.NORMAL = "\033[0m" - self.DIM = "\033[2m" + self.initialize_vt100_colors() return True return False @@ -681,7 +696,7 @@ def colorize(self, error: str) -> str: """Colorize an output line by highlighting the status and error code.""" if ": error:" in error: loc, msg = error.split("error:", maxsplit=1) - if not self.show_error_codes: + if self.hide_error_codes: return ( loc + self.style("error:", "red", bold=True) + self.highlight_quote_groups(msg) ) @@ -769,9 +784,10 @@ def format_error( return self.style(msg, "red", bold=True) -def is_typeshed_file(file: str) -> bool: +def is_typeshed_file(typeshed_dir: str | None, file: str) -> bool: + typeshed_dir = typeshed_dir if typeshed_dir is not None else TYPESHED_DIR try: - return os.path.commonpath((TYPESHED_DIR, os.path.abspath(file))) == TYPESHED_DIR + return os.path.commonpath((typeshed_dir, os.path.abspath(file))) == typeshed_dir except ValueError: # Different drives on Windows return False diff --git a/mypy/version.py b/mypy/version.py index e0dc42b478f8..fa4e0c0cc58f 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "0.NNN". # - Dev versions have the form "0.NNN+dev" (PLUS sign to conform to PEP 440). # - For 1.0 we'll switch back to 1.2.3 form. -__version__ = "0.980+dev" +__version__ = "0.990" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) diff --git a/mypy/visitor.py b/mypy/visitor.py index 62e7b4f90c8e..c5aa3caa8295 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -355,7 +355,8 @@ class NodeVisitor(Generic[T], ExpressionVisitor[T], StatementVisitor[T], Pattern methods. As all methods defined here return None by default, subclasses do not always need to override all the methods. - TODO make the default return value explicit + TODO: make the default return value explicit, then turn on + empty body checking in mypy_self_check.ini. """ # Not in superclasses: diff --git a/mypy_bootstrap.ini b/mypy_bootstrap.ini index 3a6eee6449d2..c680990fbd9e 100644 --- a/mypy_bootstrap.ini +++ b/mypy_bootstrap.ini @@ -13,3 +13,7 @@ warn_redundant_casts = True warn_unused_configs = True show_traceback = True always_true = MYPYC + +[mypy-mypy.visitor] +# See docstring for NodeVisitor for motivation. +disable_error_code = empty-body diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 5dc497528fab..719148240c89 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -1,30 +1,17 @@ [mypy] -warn_unused_configs = True -disallow_any_generics = True -disallow_subclassing_any = True -disallow_untyped_calls = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_redundant_casts = True -warn_unused_ignores = True -no_implicit_reexport = True -strict_equality = True -strict_concatenate = True - -; This is the only setting in --strict that we don't have enabled -warn_return_any = False - +strict = True warn_no_return = True strict_optional = True disallow_any_unimported = True show_traceback = True -show_error_codes = True pretty = True always_false = MYPYC plugins = misc/proper_plugin.py python_version = 3.7 exclude = mypy/typeshed/|mypyc/test-data/|mypyc/lib-rt/ +enable_error_code = ignore-without-code,redundant-expr + +[mypy-mypy.visitor] +# See docstring for NodeVisitor for motivation. +disable_error_code = empty-body diff --git a/mypyc/analysis/attrdefined.py b/mypyc/analysis/attrdefined.py index 170c0029ba04..dc871a93eba1 100644 --- a/mypyc/analysis/attrdefined.py +++ b/mypyc/analysis/attrdefined.py @@ -91,7 +91,7 @@ def foo(self) -> int: SetMem, Unreachable, ) -from mypyc.ir.rtypes import RInstance +from mypyc.ir.rtypes import RInstance, is_fixed_width_rtype # If True, print out all always-defined attributes of native classes (to aid # debugging and testing) @@ -120,6 +120,11 @@ def analyze_always_defined_attrs(class_irs: list[ClassIR]) -> None: for cl in class_irs: update_always_defined_attrs_using_subclasses(cl, seen) + # Final pass: detect attributes that need to use a bitmap to track definedness + seen = set() + for cl in class_irs: + detect_undefined_bitmap(cl, seen) + def analyze_always_defined_attrs_in_class(cl: ClassIR, seen: set[ClassIR]) -> None: if cl in seen: @@ -407,3 +412,17 @@ def update_always_defined_attrs_using_subclasses(cl: ClassIR, seen: set[ClassIR] removed.add(attr) cl._always_initialized_attrs -= removed seen.add(cl) + + +def detect_undefined_bitmap(cl: ClassIR, seen: Set[ClassIR]) -> None: + if cl in seen: + return + seen.add(cl) + for base in cl.base_mro[1:]: + detect_undefined_bitmap(cl, seen) + + if len(cl.base_mro) > 1: + cl.bitmap_attrs.extend(cl.base_mro[1].bitmap_attrs) + for n, t in cl.attributes.items(): + if is_fixed_width_rtype(t) and not cl.is_always_defined(n): + cl.bitmap_attrs.append(n) diff --git a/mypyc/analysis/dataflow.py b/mypyc/analysis/dataflow.py index 824d64a1bf4b..21c4da8981d1 100644 --- a/mypyc/analysis/dataflow.py +++ b/mypyc/analysis/dataflow.py @@ -68,7 +68,7 @@ def __init__( def __str__(self) -> str: lines = [] - lines.append("exits: %s" % sorted(self.exits, key=lambda e: e.label)) + lines.append("exits: %s" % sorted(self.exits, key=lambda e: int(e.label))) lines.append("succ: %s" % self.succ) lines.append("pred: %s" % self.pred) return "\n".join(lines) diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index c2cdd073f62e..b141784ef9ff 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -129,7 +129,11 @@ def check_op_sources_valid(fn: FuncIR) -> list[FnError]: for block in fn.blocks: valid_ops.update(block.ops) - valid_registers.update([op.dest for op in block.ops if isinstance(op, BaseAssign)]) + for op in block.ops: + if isinstance(op, BaseAssign): + valid_registers.add(op.dest) + elif isinstance(op, LoadAddress) and isinstance(op.src, Register): + valid_registers.add(op.src) valid_registers.update(fn.arg_regs) @@ -150,7 +154,7 @@ def check_op_sources_valid(fn: FuncIR) -> list[FnError]: if source not in valid_registers: errors.append( FnError( - source=op, desc=f"Invalid op reference to register {source.name}" + source=op, desc=f"Invalid op reference to register {source.name!r}" ) ) diff --git a/mypyc/build.py b/mypyc/build.py index db548b149946..4f40a6cd0865 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -533,10 +533,8 @@ def mypycify( "-Wno-unused-variable", "-Wno-unused-command-line-argument", "-Wno-unknown-warning-option", + "-Wno-unused-but-set-variable", ] - if "gcc" in compiler.compiler[0] or "gnu-cc" in compiler.compiler[0]: - # This flag is needed for gcc but does not exist on clang. - cflags += ["-Wno-unused-but-set-variable"] elif compiler.compiler_type == "msvc": # msvc doesn't have levels, '/O2' is full and '/Od' is disable if opt_level == "0": diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 3fd48dcd1cb8..5d47636b4c1e 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -9,6 +9,7 @@ from mypyc.codegen.literals import Literals from mypyc.common import ( ATTR_PREFIX, + BITMAP_BITS, FAST_ISINSTANCE_MAX_SUBCLASSES, NATIVE_PREFIX, REG_PREFIX, @@ -329,21 +330,81 @@ def tuple_c_declaration(self, rtuple: RTuple) -> list[str]: return result + def bitmap_field(self, index: int) -> str: + """Return C field name used for attribute bitmap.""" + n = index // BITMAP_BITS + if n == 0: + return "bitmap" + return f"bitmap{n + 1}" + + def attr_bitmap_expr(self, obj: str, cl: ClassIR, index: int) -> str: + """Return reference to the attribute definedness bitmap.""" + cast = f"({cl.struct_name(self.names)} *)" + attr = self.bitmap_field(index) + return f"({cast}{obj})->{attr}" + + def emit_attr_bitmap_set( + self, value: str, obj: str, rtype: RType, cl: ClassIR, attr: str + ) -> None: + """Mark an attribute as defined in the attribute bitmap. + + Assumes that the attribute is tracked in the bitmap (only some attributes + use the bitmap). If 'value' is not equal to the error value, do nothing. + """ + self._emit_attr_bitmap_update(value, obj, rtype, cl, attr, clear=False) + + def emit_attr_bitmap_clear(self, obj: str, rtype: RType, cl: ClassIR, attr: str) -> None: + """Mark an attribute as undefined in the attribute bitmap. + + Unlike emit_attr_bitmap_set, clear unconditionally. + """ + self._emit_attr_bitmap_update("", obj, rtype, cl, attr, clear=True) + + def _emit_attr_bitmap_update( + self, value: str, obj: str, rtype: RType, cl: ClassIR, attr: str, clear: bool + ) -> None: + if value: + self.emit_line(f"if (unlikely({value} == {self.c_undefined_value(rtype)})) {{") + index = cl.bitmap_attrs.index(attr) + mask = 1 << (index & (BITMAP_BITS - 1)) + bitmap = self.attr_bitmap_expr(obj, cl, index) + if clear: + self.emit_line(f"{bitmap} &= ~{mask};") + else: + self.emit_line(f"{bitmap} |= {mask};") + if value: + self.emit_line("}") + def use_vectorcall(self) -> bool: return use_vectorcall(self.capi_version) def emit_undefined_attr_check( - self, rtype: RType, attr_expr: str, compare: str, unlikely: bool = False + self, + rtype: RType, + attr_expr: str, + compare: str, + obj: str, + attr: str, + cl: ClassIR, + *, + unlikely: bool = False, ) -> None: if isinstance(rtype, RTuple): - check = "({})".format( + check = "{}".format( self.tuple_undefined_check_cond(rtype, attr_expr, self.c_undefined_value, compare) ) else: - check = f"({attr_expr} {compare} {self.c_undefined_value(rtype)})" + undefined = self.c_undefined_value(rtype) + check = f"{attr_expr} {compare} {undefined}" if unlikely: - check = f"(unlikely{check})" - self.emit_line(f"if {check} {{") + check = f"unlikely({check})" + if is_fixed_width_rtype(rtype): + index = cl.bitmap_attrs.index(attr) + bit = 1 << (index & (BITMAP_BITS - 1)) + attr = self.bitmap_field(index) + obj_expr = f"({cl.struct_name(self.names)} *){obj}" + check = f"{check} && !(({obj_expr})->{attr} & {bit})" + self.emit_line(f"if ({check}) {{") def tuple_undefined_check_cond( self, @@ -925,7 +986,11 @@ def emit_box( def emit_error_check(self, value: str, rtype: RType, failure: str) -> None: """Emit code for checking a native function return value for uncaught exception.""" - if not isinstance(rtype, RTuple): + if is_fixed_width_rtype(rtype): + # The error value is also valid as a normal value, so we need to also check + # for a raised exception. + self.emit_line(f"if ({value} == {self.c_error_value(rtype)} && PyErr_Occurred()) {{") + elif not isinstance(rtype, RTuple): self.emit_line(f"if ({value} == {self.c_error_value(rtype)}) {{") else: if len(rtype.types) == 0: diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 5434b5c01219..0fdb6e8a98c3 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -17,10 +17,10 @@ generate_richcompare_wrapper, generate_set_del_item_wrapper, ) -from mypyc.common import NATIVE_PREFIX, PREFIX, REG_PREFIX, use_fastcall +from mypyc.common import BITMAP_BITS, BITMAP_TYPE, NATIVE_PREFIX, PREFIX, REG_PREFIX, use_fastcall from mypyc.ir.class_ir import ClassIR, VTableEntries from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR -from mypyc.ir.rtypes import RTuple, RType, object_rprimitive +from mypyc.ir.rtypes import RTuple, RType, is_fixed_width_rtype, object_rprimitive from mypyc.namegen import NameGenerator from mypyc.sametype import is_same_type @@ -61,11 +61,15 @@ def wrapper_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: AS_SEQUENCE_SLOT_DEFS: SlotTable = {"__contains__": ("sq_contains", generate_contains_wrapper)} AS_NUMBER_SLOT_DEFS: SlotTable = { + # Unary operations. "__bool__": ("nb_bool", generate_bool_wrapper), - "__neg__": ("nb_negative", generate_dunder_wrapper), - "__invert__": ("nb_invert", generate_dunder_wrapper), "__int__": ("nb_int", generate_dunder_wrapper), "__float__": ("nb_float", generate_dunder_wrapper), + "__neg__": ("nb_negative", generate_dunder_wrapper), + "__pos__": ("nb_positive", generate_dunder_wrapper), + "__abs__": ("nb_absolute", generate_dunder_wrapper), + "__invert__": ("nb_invert", generate_dunder_wrapper), + # Binary operations. "__add__": ("nb_add", generate_bin_op_wrapper), "__radd__": ("nb_add", generate_bin_op_wrapper), "__sub__": ("nb_subtract", generate_bin_op_wrapper), @@ -90,6 +94,7 @@ def wrapper_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: "__rxor__": ("nb_xor", generate_bin_op_wrapper), "__matmul__": ("nb_matrix_multiply", generate_bin_op_wrapper), "__rmatmul__": ("nb_matrix_multiply", generate_bin_op_wrapper), + # In-place binary operations. "__iadd__": ("nb_inplace_add", generate_dunder_wrapper), "__isub__": ("nb_inplace_subtract", generate_dunder_wrapper), "__imul__": ("nb_inplace_multiply", generate_dunder_wrapper), @@ -367,8 +372,17 @@ def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None: lines += ["typedef struct {", "PyObject_HEAD", "CPyVTableItem *vtable;"] if cl.has_method("__call__") and emitter.use_vectorcall(): lines.append("vectorcallfunc vectorcall;") + bitmap_attrs = [] for base in reversed(cl.base_mro): if not base.is_trait: + if base.bitmap_attrs: + # Do we need another attribute bitmap field? + if emitter.bitmap_field(len(base.bitmap_attrs) - 1) not in bitmap_attrs: + for i in range(0, len(base.bitmap_attrs), BITMAP_BITS): + attr = emitter.bitmap_field(i) + if attr not in bitmap_attrs: + lines.append(f"{BITMAP_TYPE} {attr};") + bitmap_attrs.append(attr) for attr, rtype in base.attributes.items(): if (attr, rtype) not in seen_attrs: lines.append(f"{emitter.ctype_spaced(rtype)}{emitter.attr(attr)};") @@ -546,6 +560,9 @@ def generate_setup_for_class( emitter.emit_line("}") else: emitter.emit_line(f"self->vtable = {vtable_name};") + for i in range(0, len(cl.bitmap_attrs), BITMAP_BITS): + field = emitter.bitmap_field(i) + emitter.emit_line(f"self->{field} = 0;") if cl.has_method("__call__") and emitter.use_vectorcall(): name = cl.method_decl("__call__").cname(emitter.names) @@ -887,7 +904,7 @@ def generate_getter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N always_defined = cl.is_always_defined(attr) and not rtype.is_refcounted if not always_defined: - emitter.emit_undefined_attr_check(rtype, attr_expr, "==", unlikely=True) + emitter.emit_undefined_attr_check(rtype, attr_expr, "==", "self", attr, cl, unlikely=True) emitter.emit_line("PyErr_SetString(PyExc_AttributeError,") emitter.emit_line(f' "attribute {repr(attr)} of {repr(cl.name)} undefined");') emitter.emit_line("return NULL;") @@ -926,7 +943,7 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N if rtype.is_refcounted: attr_expr = f"self->{attr_field}" if not always_defined: - emitter.emit_undefined_attr_check(rtype, attr_expr, "!=") + emitter.emit_undefined_attr_check(rtype, attr_expr, "!=", "self", attr, cl) emitter.emit_dec_ref(f"self->{attr_field}", rtype) if not always_defined: emitter.emit_line("}") @@ -943,9 +960,14 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N emitter.emit_lines("if (!tmp)", " return -1;") emitter.emit_inc_ref("tmp", rtype) emitter.emit_line(f"self->{attr_field} = tmp;") + if is_fixed_width_rtype(rtype) and not always_defined: + emitter.emit_attr_bitmap_set("tmp", "self", rtype, cl, attr) + if deletable: emitter.emit_line("} else") emitter.emit_line(f" self->{attr_field} = {emitter.c_undefined_value(rtype)};") + if is_fixed_width_rtype(rtype): + emitter.emit_attr_bitmap_clear("self", rtype, cl, attr) emitter.emit_line("return 0;") emitter.emit_line("}") diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index c0aaff2c5f99..2c096655f41e 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -60,6 +60,7 @@ RStruct, RTuple, RType, + is_fixed_width_rtype, is_int32_rprimitive, is_int64_rprimitive, is_int_rprimitive, @@ -353,7 +354,9 @@ def visit_get_attr(self, op: GetAttr) -> None: always_defined = cl.is_always_defined(op.attr) merged_branch = None if not always_defined: - self.emitter.emit_undefined_attr_check(attr_rtype, dest, "==", unlikely=True) + self.emitter.emit_undefined_attr_check( + attr_rtype, dest, "==", obj, op.attr, cl, unlikely=True + ) branch = self.next_branch() if branch is not None: if ( @@ -433,10 +436,17 @@ def visit_set_attr(self, op: SetAttr) -> None: # previously undefined), so decref the old value. always_defined = cl.is_always_defined(op.attr) if not always_defined: - self.emitter.emit_undefined_attr_check(attr_rtype, attr_expr, "!=") + self.emitter.emit_undefined_attr_check( + attr_rtype, attr_expr, "!=", obj, op.attr, cl + ) self.emitter.emit_dec_ref(attr_expr, attr_rtype) if not always_defined: self.emitter.emit_line("}") + elif is_fixed_width_rtype(attr_rtype) and not cl.is_always_defined(op.attr): + # If there is overlap with the error value, update bitmap to mark + # attribute as defined. + self.emitter.emit_attr_bitmap_set(src, obj, attr_rtype, cl, op.attr) + # This steals the reference to src, so we don't need to increment the arg self.emitter.emit_line(f"{attr_expr} = {src};") if op.error_kind == ERR_FALSE: diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 005c0f764e9a..5dacaf6acab6 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -409,7 +409,6 @@ def compile_modules_to_c( compiler_options: The compilation options errors: Where to report any errors encountered groups: The groups that we are compiling. See documentation of Groups type above. - ops: Optionally, where to dump stringified ops for debugging. Returns the IR of the modules and a list containing the generated files for each group. """ @@ -419,7 +418,9 @@ def compile_modules_to_c( # Sometimes when we call back into mypy, there might be errors. # We don't want to crash when that happens. - result.manager.errors.set_file("", module=None, scope=None) + result.manager.errors.set_file( + "", module=None, scope=None, options=result.manager.options + ) modules = compile_modules_to_ir(result, mapper, compiler_options, errors) ctext = compile_ir_to_c(groups, modules, result, mapper, compiler_options) diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index a296ce271d07..1abab53bc39d 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -17,13 +17,22 @@ from mypy.nodes import ARG_NAMED, ARG_NAMED_OPT, ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, ArgKind from mypy.operators import op_methods_to_symbols, reverse_op_method_names, reverse_op_methods from mypyc.codegen.emit import AssignHandler, Emitter, ErrorHandler, GotoHandler, ReturnHandler -from mypyc.common import DUNDER_PREFIX, NATIVE_PREFIX, PREFIX, use_vectorcall +from mypyc.common import ( + BITMAP_BITS, + BITMAP_TYPE, + DUNDER_PREFIX, + NATIVE_PREFIX, + PREFIX, + bitmap_name, + use_vectorcall, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncIR, RuntimeArg from mypyc.ir.rtypes import ( RInstance, RType, is_bool_rprimitive, + is_fixed_width_rtype, is_int_rprimitive, is_object_rprimitive, object_rprimitive, @@ -135,6 +144,8 @@ def generate_wrapper_function( # If fn is a method, then the first argument is a self param real_args = list(fn.args) + if fn.sig.num_bitmap_args: + real_args = real_args[: -fn.sig.num_bitmap_args] if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD: arg = real_args.pop(0) emitter.emit_line(f"PyObject *obj_{arg.name} = self;") @@ -185,6 +196,9 @@ def generate_wrapper_function( "return NULL;", "}", ) + for i in range(fn.sig.num_bitmap_args): + name = bitmap_name(i) + emitter.emit_line(f"{BITMAP_TYPE} {name} = 0;") traceback_code = generate_traceback_code(fn, emitter, source_path, module_name) generate_wrapper_core( fn, @@ -223,6 +237,8 @@ def generate_legacy_wrapper_function( # If fn is a method, then the first argument is a self param real_args = list(fn.args) + if fn.sig.num_bitmap_args: + real_args = real_args[: -fn.sig.num_bitmap_args] if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD: arg = real_args.pop(0) emitter.emit_line(f"PyObject *obj_{arg.name} = self;") @@ -254,6 +270,9 @@ def generate_legacy_wrapper_function( "return NULL;", "}", ) + for i in range(fn.sig.num_bitmap_args): + name = bitmap_name(i) + emitter.emit_line(f"{BITMAP_TYPE} {name} = 0;") traceback_code = generate_traceback_code(fn, emitter, source_path, module_name) generate_wrapper_core( fn, @@ -669,7 +688,8 @@ def generate_wrapper_core( """ gen = WrapperGenerator(None, emitter) gen.set_target(fn) - gen.arg_names = arg_names or [arg.name for arg in fn.args] + if arg_names: + gen.arg_names = arg_names gen.cleanups = cleanups or [] gen.optional_args = optional_args or [] gen.traceback_code = traceback_code or "" @@ -688,6 +708,7 @@ def generate_arg_check( *, optional: bool = False, raise_exception: bool = True, + bitmap_arg_index: int = 0, ) -> None: """Insert a runtime check for argument and unbox if necessary. @@ -697,17 +718,34 @@ def generate_arg_check( """ error = error or AssignHandler() if typ.is_unboxed: - # Borrow when unboxing to avoid reference count manipulation. - emitter.emit_unbox( - f"obj_{name}", - f"arg_{name}", - typ, - declare_dest=True, - raise_exception=raise_exception, - error=error, - borrow=True, - optional=optional, - ) + if is_fixed_width_rtype(typ) and optional: + # Update bitmap is value is provided. + emitter.emit_line(f"{emitter.ctype(typ)} arg_{name} = 0;") + emitter.emit_line(f"if (obj_{name} != NULL) {{") + bitmap = bitmap_name(bitmap_arg_index // BITMAP_BITS) + emitter.emit_line(f"{bitmap} |= 1 << {bitmap_arg_index & (BITMAP_BITS - 1)};") + emitter.emit_unbox( + f"obj_{name}", + f"arg_{name}", + typ, + declare_dest=False, + raise_exception=raise_exception, + error=error, + borrow=True, + ) + emitter.emit_line("}") + else: + # Borrow when unboxing to avoid reference count manipulation. + emitter.emit_unbox( + f"obj_{name}", + f"arg_{name}", + typ, + declare_dest=True, + raise_exception=raise_exception, + error=error, + borrow=True, + optional=optional, + ) elif is_object_rprimitive(typ): # Object is trivial since any object is valid if optional: @@ -749,8 +787,12 @@ def set_target(self, fn: FuncIR) -> None: """ self.target_name = fn.name self.target_cname = fn.cname(self.emitter.names) - self.arg_names = [arg.name for arg in fn.args] - self.args = fn.args[:] + self.num_bitmap_args = fn.sig.num_bitmap_args + if self.num_bitmap_args: + self.args = fn.args[: -self.num_bitmap_args] + else: + self.args = fn.args + self.arg_names = [arg.name for arg in self.args] self.ret_type = fn.ret_type def wrapper_name(self) -> str: @@ -779,17 +821,22 @@ def emit_arg_processing( ) -> None: """Emit validation and unboxing of arguments.""" error = error or self.error() + bitmap_arg_index = 0 for arg_name, arg in zip(self.arg_names, self.args): # Suppress the argument check for *args/**kwargs, since we know it must be right. typ = arg.type if arg.kind not in (ARG_STAR, ARG_STAR2) else object_rprimitive + optional = arg in self.optional_args generate_arg_check( arg_name, typ, self.emitter, error, raise_exception=raise_exception, - optional=arg in self.optional_args, + optional=optional, + bitmap_arg_index=bitmap_arg_index, ) + if optional and is_fixed_width_rtype(typ): + bitmap_arg_index += 1 def emit_call(self, not_implemented_handler: str = "") -> None: """Emit call to the wrapper function. @@ -798,6 +845,12 @@ def emit_call(self, not_implemented_handler: str = "") -> None: a NotImplemented return value (if it's possible based on the return type). """ native_args = ", ".join(f"arg_{arg}" for arg in self.arg_names) + if self.num_bitmap_args: + bitmap_args = ", ".join( + [bitmap_name(i) for i in reversed(range(self.num_bitmap_args))] + ) + native_args = f"{native_args}, {bitmap_args}" + ret_type = self.ret_type emitter = self.emitter if ret_type.is_unboxed or self.use_goto(): diff --git a/mypyc/common.py b/mypyc/common.py index e9b59246898b..6b0bbcee5fc9 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +import sysconfig from typing import Any, Dict from typing_extensions import Final @@ -30,7 +31,16 @@ # Maximal number of subclasses for a class to trigger fast path in isinstance() checks. FAST_ISINSTANCE_MAX_SUBCLASSES: Final = 2 -IS_32_BIT_PLATFORM: Final = sys.maxsize < (1 << 31) +# Size of size_t, if configured. +SIZEOF_SIZE_T_SYSCONFIG: Final = sysconfig.get_config_var("SIZEOF_SIZE_T") + +SIZEOF_SIZE_T: Final = ( + int(SIZEOF_SIZE_T_SYSCONFIG) + if SIZEOF_SIZE_T_SYSCONFIG is not None + else (sys.maxsize + 1).bit_length() // 8 +) + +IS_32_BIT_PLATFORM: Final = int(SIZEOF_SIZE_T) == 4 PLATFORM_SIZE = 4 if IS_32_BIT_PLATFORM else 8 @@ -42,15 +52,25 @@ IS_MIXED_32_64_BIT_BUILD: Final = sys.platform in ["darwin"] and sys.version_info < (3, 6) # Maximum value for a short tagged integer. -MAX_SHORT_INT: Final = sys.maxsize >> 1 +MAX_SHORT_INT: Final = 2 ** (8 * int(SIZEOF_SIZE_T) - 2) - 1 + +# Minimum value for a short tagged integer. +MIN_SHORT_INT: Final = -(MAX_SHORT_INT) - 1 # Maximum value for a short tagged integer represented as a C integer literal. # # Note: Assume that the compiled code uses the same bit width as mypyc, except for # Python 3.5 on macOS. -MAX_LITERAL_SHORT_INT: Final = sys.maxsize >> 1 if not IS_MIXED_32_64_BIT_BUILD else 2**30 - 1 +MAX_LITERAL_SHORT_INT: Final = MAX_SHORT_INT if not IS_MIXED_32_64_BIT_BUILD else 2**30 - 1 MIN_LITERAL_SHORT_INT: Final = -MAX_LITERAL_SHORT_INT - 1 +# Decription of the C type used to track the definedness of attributes and +# the presence of argument default values that have types with overlapping +# error values. Each tracked attribute/argument has a dedicated bit in the +# relevant bitmap. +BITMAP_TYPE: Final = "uint32_t" +BITMAP_BITS: Final = 32 + # Runtime C library files RUNTIME_C_FILES: Final = [ "init.c", @@ -121,3 +141,9 @@ def short_id_from_name(func_name: str, shortname: str, line: int | None) -> str: else: partial_name = shortname return partial_name + + +def bitmap_name(index: int) -> str: + if index == 0: + return "__bitmap" + return f"__bitmap{index + 1}" diff --git a/mypyc/doc/conf.py b/mypyc/doc/conf.py index 2077c04f093c..da887e0d8267 100644 --- a/mypyc/doc/conf.py +++ b/mypyc/doc/conf.py @@ -36,7 +36,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] # type: ignore +extensions = [] # type: ignore[var-annotated] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/mypyc/doc/native_operations.rst b/mypyc/doc/native_operations.rst index 896217063fee..2587e982feac 100644 --- a/mypyc/doc/native_operations.rst +++ b/mypyc/doc/native_operations.rst @@ -24,6 +24,7 @@ Functions * ``cast(, obj)`` * ``type(obj)`` * ``len(obj)`` +* ``abs(obj)`` * ``id(obj)`` * ``iter(obj)`` * ``next(iter: Iterator)`` diff --git a/mypyc/errors.py b/mypyc/errors.py index 2fb07c10827a..1dd269fe25f3 100644 --- a/mypyc/errors.py +++ b/mypyc/errors.py @@ -7,7 +7,7 @@ class Errors: def __init__(self) -> None: self.num_errors = 0 self.num_warnings = 0 - self._errors = mypy.errors.Errors() + self._errors = mypy.errors.Errors(hide_error_codes=True) def error(self, msg: str, path: str, line: int) -> None: self._errors.report(line, None, msg, severity="error", file=path) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index dca19e5a2e3c..7f55decfd754 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -130,7 +130,7 @@ def __init__( self.builtin_base: str | None = None # Default empty constructor self.ctor = FuncDecl(name, None, module_name, FuncSignature([], RInstance(self))) - + # Attributes defined in the class (not inherited) self.attributes: dict[str, RType] = {} # Deletable attributes self.deletable: list[str] = [] @@ -184,6 +184,13 @@ def __init__( # If True, __init__ can make 'self' visible to unanalyzed/arbitrary code self.init_self_leak = False + # Definedness of these attributes is backed by a bitmap. Index in the list + # indicates the bit number. Includes inherited attributes. We need the + # bitmap for types such as native ints that can't have a dedicated error + # value that doesn't overlap a valid value. The bitmap is used if the + # value of an attribute is the same as the error value. + self.bitmap_attrs: List[str] = [] + def __repr__(self) -> str: return ( "ClassIR(" diff --git a/mypyc/ir/func_ir.py b/mypyc/ir/func_ir.py index 82ce23402d10..dc83de24300a 100644 --- a/mypyc/ir/func_ir.py +++ b/mypyc/ir/func_ir.py @@ -6,7 +6,7 @@ from typing_extensions import Final from mypy.nodes import ARG_POS, ArgKind, Block, FuncDef -from mypyc.common import JsonDict, get_id_from_name, short_id_from_name +from mypyc.common import BITMAP_BITS, JsonDict, bitmap_name, get_id_from_name, short_id_from_name from mypyc.ir.ops import ( Assign, AssignMulti, @@ -17,7 +17,7 @@ Register, Value, ) -from mypyc.ir.rtypes import RType, deserialize_type +from mypyc.ir.rtypes import RType, bitmap_rprimitive, deserialize_type, is_fixed_width_rtype from mypyc.namegen import NameGenerator @@ -70,12 +70,37 @@ class FuncSignature: def __init__(self, args: Sequence[RuntimeArg], ret_type: RType) -> None: self.args = tuple(args) self.ret_type = ret_type + # Bitmap arguments are use to mark default values for arguments that + # have types with overlapping error values. + self.num_bitmap_args = num_bitmap_args(self.args) + if self.num_bitmap_args: + extra = [ + RuntimeArg(bitmap_name(i), bitmap_rprimitive, pos_only=True) + for i in range(self.num_bitmap_args) + ] + self.args = self.args + tuple(reversed(extra)) + + def real_args(self) -> tuple[RuntimeArg, ...]: + """Return arguments without any synthetic bitmap arguments.""" + if self.num_bitmap_args: + return self.args[: -self.num_bitmap_args] + return self.args + + def bound_sig(self) -> "FuncSignature": + if self.num_bitmap_args: + return FuncSignature(self.args[1 : -self.num_bitmap_args], self.ret_type) + else: + return FuncSignature(self.args[1:], self.ret_type) def __repr__(self) -> str: return f"FuncSignature(args={self.args!r}, ret={self.ret_type!r})" def serialize(self) -> JsonDict: - return {"args": [t.serialize() for t in self.args], "ret_type": self.ret_type.serialize()} + if self.num_bitmap_args: + args = self.args[: -self.num_bitmap_args] + else: + args = self.args + return {"args": [t.serialize() for t in args], "ret_type": self.ret_type.serialize()} @classmethod def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncSignature: @@ -85,6 +110,14 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> FuncSignature: ) +def num_bitmap_args(args: tuple[RuntimeArg, ...]) -> int: + n = 0 + for arg in args: + if is_fixed_width_rtype(arg.type) and arg.kind.is_optional(): + n += 1 + return (n + (BITMAP_BITS - 1)) // BITMAP_BITS + + FUNC_NORMAL: Final = 0 FUNC_STATICMETHOD: Final = 1 FUNC_CLASSMETHOD: Final = 2 @@ -120,7 +153,7 @@ def __init__( if kind == FUNC_STATICMETHOD: self.bound_sig = sig else: - self.bound_sig = FuncSignature(sig.args[1:], sig.ret_type) + self.bound_sig = sig.bound_sig() # this is optional because this will be set to the line number when the corresponding # FuncIR is created diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 56a1e6103acf..361221f5b710 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -633,7 +633,7 @@ def __init__(self, obj: Value, attr: str, line: int, *, borrow: bool = False) -> attr_type = obj.type.attr_type(attr) self.type = attr_type if is_fixed_width_rtype(attr_type): - self.error_kind = ERR_NEVER + self.error_kind = ERR_MAGIC_OVERLAPPING self.is_borrowed = borrow and attr_type.is_refcounted def sources(self) -> list[Value]: diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index 0ef555f86738..a9324a8608e4 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -119,6 +119,7 @@ def borrow_prefix(self, op: Op) -> str: def visit_set_attr(self, op: SetAttr) -> str: if op.is_init: assert op.error_kind == ERR_NEVER + if op.error_kind == ERR_NEVER: # Initialization and direct struct access can never fail return self.format("%r.%s = %r", op.obj, op.attr, op.src) else: diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 9b023da24443..6db3f249ca9b 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -361,6 +361,9 @@ def __hash__(self) -> int: "c_ptr", is_unboxed=False, is_refcounted=False, ctype="void *" ) +# The type corresponding to mypyc.common.BITMAP_TYPE +bitmap_rprimitive: Final = uint32_rprimitive + # Floats are represent as 'float' PyObject * values. (In the future # we'll likely switch to a more efficient, unboxed representation.) float_rprimitive: Final = RPrimitive("builtins.float", is_unboxed=False, is_refcounted=True) diff --git a/mypyc/irbuild/ast_helpers.py b/mypyc/irbuild/ast_helpers.py index 5b0c58717301..1af1ad611a89 100644 --- a/mypyc/irbuild/ast_helpers.py +++ b/mypyc/irbuild/ast_helpers.py @@ -21,7 +21,7 @@ Var, ) from mypyc.ir.ops import BasicBlock -from mypyc.ir.rtypes import is_tagged +from mypyc.ir.rtypes import is_fixed_width_rtype, is_tagged from mypyc.irbuild.builder import IRBuilder from mypyc.irbuild.constant_fold import constant_fold_expr @@ -70,7 +70,10 @@ def maybe_process_conditional_comparison( return False ltype = self.node_type(e.operands[0]) rtype = self.node_type(e.operands[1]) - if not is_tagged(ltype) or not is_tagged(rtype): + if not ( + (is_tagged(ltype) or is_fixed_width_rtype(ltype)) + and (is_tagged(rtype) or is_fixed_width_rtype(rtype)) + ): return False op = e.operators[0] if op not in ("==", "!=", "<", "<=", ">", ">="): @@ -80,8 +83,17 @@ def maybe_process_conditional_comparison( borrow_left = is_borrow_friendly_expr(self, right_expr) left = self.accept(left_expr, can_borrow=borrow_left) right = self.accept(right_expr, can_borrow=True) - # "left op right" for two tagged integers - self.builder.compare_tagged_condition(left, right, op, true, false, e.line) + if is_fixed_width_rtype(ltype) or is_fixed_width_rtype(rtype): + if not is_fixed_width_rtype(ltype): + left = self.coerce(left, rtype, e.line) + elif not is_fixed_width_rtype(rtype): + right = self.coerce(right, ltype, e.line) + reg = self.binary_op(left, right, op, e.line) + self.builder.flush_keep_alives() + self.add_bool_branch(reg, true, false) + else: + # "left op right" for two tagged integers + self.builder.compare_tagged_condition(left, right, op, true, false, e.line) return True diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index cde12e2a0a75..443fa6886ea6 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -45,10 +45,19 @@ UnaryExpr, Var, ) -from mypy.types import Instance, TupleType, Type, UninhabitedType, get_proper_type +from mypy.types import ( + AnyType, + Instance, + ProperType, + TupleType, + Type, + TypeOfAny, + UninhabitedType, + get_proper_type, +) from mypy.util import split_target from mypy.visitor import ExpressionVisitor, StatementVisitor -from mypyc.common import SELF_NAME, TEMP_ATTR_NAME +from mypyc.common import BITMAP_BITS, SELF_NAME, TEMP_ATTR_NAME from mypyc.crash import catch_errors from mypyc.errors import Errors from mypyc.ir.class_ir import ClassIR, NonExtClassInfo @@ -58,9 +67,11 @@ Assign, BasicBlock, Branch, + ComparisonOp, GetAttr, InitStatic, Integer, + IntOp, LoadStatic, Op, RaiseStandardError, @@ -74,10 +85,12 @@ RInstance, RTuple, RType, + bitmap_rprimitive, c_int_rprimitive, c_pyssize_t_rprimitive, dict_rprimitive, int_rprimitive, + is_fixed_width_rtype, is_list_rprimitive, is_none_rprimitive, is_object_rprimitive, @@ -446,6 +459,24 @@ def assign_if_null(self, target: Register, get_val: Callable[[], Value], line: i self.goto(body_block) self.activate_block(body_block) + def assign_if_bitmap_unset( + self, target: Register, get_val: Callable[[], Value], index: int, line: int + ) -> None: + error_block, body_block = BasicBlock(), BasicBlock() + o = self.int_op( + bitmap_rprimitive, + self.builder.args[-1 - index // BITMAP_BITS], + Integer(1 << (index & (BITMAP_BITS - 1)), bitmap_rprimitive), + IntOp.AND, + line, + ) + b = self.add(ComparisonOp(o, Integer(0, bitmap_rprimitive), ComparisonOp.EQ)) + self.add(Branch(b, error_block, body_block, Branch.BOOL)) + self.activate_block(error_block) + self.add(Assign(target, self.coerce(get_val(), target.type, line))) + self.goto(body_block) + self.activate_block(body_block) + def maybe_add_implicit_return(self) -> None: if is_none_rprimitive(self.ret_types[-1]) or is_object_rprimitive(self.ret_types[-1]): self.add_implicit_return() @@ -867,7 +898,11 @@ def get_dict_item_type(self, expr: Expression) -> RType: def _analyze_iterable_item_type(self, expr: Expression) -> Type: """Return the item type given by 'expr' in an iterable context.""" # This logic is copied from mypy's TypeChecker.analyze_iterable_item_type. - iterable = get_proper_type(self.types[expr]) + if expr not in self.types: + # Mypy thinks this is unreachable. + iterable: ProperType = AnyType(TypeOfAny.from_error) + else: + iterable = get_proper_type(self.types[expr]) echk = self.graph[self.module_name].type_checker().expr_checker iterator = echk.check_method_call_by_name("__iter__", iterable, [], [], expr)[0] @@ -1246,6 +1281,7 @@ def gen_arg_defaults(builder: IRBuilder) -> None: value to the argument. """ fitem = builder.fn_info.fitem + nb = 0 for arg in fitem.arguments: if arg.initializer: target = builder.lookup(arg.variable) @@ -1271,7 +1307,14 @@ def get_default() -> Value: ) assert isinstance(target, AssignmentTargetRegister) - builder.assign_if_null(target.register, get_default, arg.initializer.line) + reg = target.register + if not is_fixed_width_rtype(reg.type): + builder.assign_if_null(target.register, get_default, arg.initializer.line) + else: + builder.assign_if_bitmap_unset( + target.register, get_default, nb, arg.initializer.line + ) + nb += 1 def remangle_redefinition_name(name: str) -> str: diff --git a/mypyc/irbuild/callable_class.py b/mypyc/irbuild/callable_class.py index 1170e3fc7363..d3ee54a208cd 100644 --- a/mypyc/irbuild/callable_class.py +++ b/mypyc/irbuild/callable_class.py @@ -92,7 +92,10 @@ def add_call_to_callable_class( given callable class, used to represent that function. """ # Since we create a method, we also add a 'self' parameter. - sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args, sig.ret_type) + nargs = len(sig.args) - sig.num_bitmap_args + sig = FuncSignature( + (RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args[:nargs], sig.ret_type + ) call_fn_decl = FuncDecl("__call__", fn_info.callable_class.ir.name, builder.module_name, sig) call_fn_ir = FuncIR( call_fn_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name diff --git a/mypyc/irbuild/constant_fold.py b/mypyc/irbuild/constant_fold.py index 08cf75d9e5ca..8d0a7fea5d90 100644 --- a/mypyc/irbuild/constant_fold.py +++ b/mypyc/irbuild/constant_fold.py @@ -80,7 +80,9 @@ def constant_fold_binary_int_op(op: str, left: int, right: int) -> int | None: return left >> right elif op == "**": if right >= 0: - return left**right + ret = left**right + assert isinstance(ret, int) + return ret return None diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index beb3215389ba..416fba633482 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -17,11 +17,11 @@ def g() -> int: from __future__ import annotations -from mypy.nodes import FuncDef, SymbolNode -from mypyc.common import ENV_ATTR_NAME, SELF_NAME +from mypy.nodes import Argument, FuncDef, SymbolNode, Var +from mypyc.common import BITMAP_BITS, ENV_ATTR_NAME, SELF_NAME, bitmap_name from mypyc.ir.class_ir import ClassIR from mypyc.ir.ops import Call, GetAttr, SetAttr, Value -from mypyc.ir.rtypes import RInstance, object_rprimitive +from mypyc.ir.rtypes import RInstance, bitmap_rprimitive, is_fixed_width_rtype, object_rprimitive from mypyc.irbuild.builder import IRBuilder, SymbolTarget from mypyc.irbuild.context import FuncInfo, GeneratorClass, ImplicitClass from mypyc.irbuild.targets import AssignmentTargetAttr @@ -159,6 +159,15 @@ def load_outer_envs(builder: IRBuilder, base: ImplicitClass) -> None: index -= 1 +def num_bitmap_args(builder: IRBuilder, args: list[Argument]) -> int: + n = 0 + for arg in args: + t = builder.type_to_rtype(arg.variable.type) + if is_fixed_width_rtype(t) and arg.kind.is_optional(): + n += 1 + return (n + (BITMAP_BITS - 1)) // BITMAP_BITS + + def add_args_to_env( builder: IRBuilder, local: bool = True, @@ -166,12 +175,16 @@ def add_args_to_env( reassign: bool = True, ) -> None: fn_info = builder.fn_info + args = fn_info.fitem.arguments + nb = num_bitmap_args(builder, args) if local: - for arg in fn_info.fitem.arguments: + for arg in args: rtype = builder.type_to_rtype(arg.variable.type) builder.add_local_reg(arg.variable, rtype, is_arg=True) + for i in reversed(range(nb)): + builder.add_local_reg(Var(bitmap_name(i)), bitmap_rprimitive, is_arg=True) else: - for arg in fn_info.fitem.arguments: + for arg in args: if is_free_variable(builder, arg.variable) or fn_info.is_generator: rtype = builder.type_to_rtype(arg.variable.type) assert base is not None, "base cannot be None for adding nonlocal args" diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index f6d488ccac42..b7d093cde7ee 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -52,6 +52,8 @@ from mypyc.ir.ops import ( Assign, BasicBlock, + ComparisonOp, + Integer, LoadAddress, RaiseStandardError, Register, @@ -62,6 +64,7 @@ from mypyc.ir.rtypes import ( RTuple, int_rprimitive, + is_fixed_width_rtype, is_int_rprimitive, is_list_rprimitive, is_none_rprimitive, @@ -472,21 +475,26 @@ def transform_op_expr(builder: IRBuilder, expr: OpExpr) -> Value: if folded: return folded + borrow_left = False + borrow_right = False + + ltype = builder.node_type(expr.left) + rtype = builder.node_type(expr.right) + # Special case some int ops to allow borrowing operands. - if is_int_rprimitive(builder.node_type(expr.left)) and is_int_rprimitive( - builder.node_type(expr.right) - ): + if is_int_rprimitive(ltype) and is_int_rprimitive(rtype): if expr.op == "//": expr = try_optimize_int_floor_divide(expr) if expr.op in int_borrow_friendly_op: borrow_left = is_borrow_friendly_expr(builder, expr.right) - left = builder.accept(expr.left, can_borrow=borrow_left) - right = builder.accept(expr.right, can_borrow=True) - return builder.binary_op(left, right, expr.op, expr.line) + borrow_right = True + elif is_fixed_width_rtype(ltype) and is_fixed_width_rtype(rtype): + borrow_left = is_borrow_friendly_expr(builder, expr.right) + borrow_right = True - return builder.binary_op( - builder.accept(expr.left), builder.accept(expr.right), expr.op, expr.line - ) + left = builder.accept(expr.left, can_borrow=borrow_left) + right = builder.accept(expr.right, can_borrow=borrow_right) + return builder.binary_op(left, right, expr.op, expr.line) def try_optimize_int_floor_divide(expr: OpExpr) -> OpExpr: @@ -708,6 +716,25 @@ def transform_basic_comparison( and op in int_comparison_op_mapping.keys() ): return builder.compare_tagged(left, right, op, line) + if is_fixed_width_rtype(left.type) and op in int_comparison_op_mapping.keys(): + if right.type == left.type: + op_id = ComparisonOp.signed_ops[op] + return builder.builder.comparison_op(left, right, op_id, line) + elif isinstance(right, Integer): + op_id = ComparisonOp.signed_ops[op] + return builder.builder.comparison_op( + left, Integer(right.value >> 1, left.type), op_id, line + ) + elif ( + is_fixed_width_rtype(right.type) + and op in int_comparison_op_mapping.keys() + and isinstance(left, Integer) + ): + op_id = ComparisonOp.signed_ops[op] + return builder.builder.comparison_op( + Integer(left.value >> 1, right.type), right, op_id, line + ) + negate = False if op == "is not": op, negate = "is", True diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 59b15423fe37..fc67178af5de 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -38,6 +38,7 @@ bool_rprimitive, int_rprimitive, is_dict_rprimitive, + is_fixed_width_rtype, is_list_rprimitive, is_sequence_rprimitive, is_short_int_rprimitive, @@ -887,7 +888,9 @@ def init(self, start_reg: Value, end_reg: Value, step: int) -> None: self.step = step self.end_target = builder.maybe_spill(end_reg) if is_short_int_rprimitive(start_reg.type) and is_short_int_rprimitive(end_reg.type): - index_type = short_int_rprimitive + index_type: RType = short_int_rprimitive + elif is_fixed_width_rtype(end_reg.type): + index_type = end_reg.type else: index_type = int_rprimitive index_reg = Register(index_type) diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index eb35a983866d..237088791bc9 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -13,7 +13,7 @@ from __future__ import annotations from collections import defaultdict -from typing import DefaultDict, NamedTuple, Sequence +from typing import NamedTuple, Sequence from mypy.nodes import ( ArgKind, @@ -89,7 +89,7 @@ from mypyc.primitives.generic_ops import py_setattr_op from mypyc.primitives.misc_ops import register_function from mypyc.primitives.registry import builtin_names -from mypyc.sametype import is_same_method_signature +from mypyc.sametype import is_same_method_signature, is_same_type # Top-level transform functions @@ -548,7 +548,7 @@ def is_decorated(builder: IRBuilder, fdef: FuncDef) -> bool: def gen_glue( builder: IRBuilder, - sig: FuncSignature, + base_sig: FuncSignature, target: FuncIR, cls: ClassIR, base: ClassIR, @@ -566,9 +566,9 @@ def gen_glue( "shadow" glue methods that work with interpreted subclasses. """ if fdef.is_property: - return gen_glue_property(builder, sig, target, cls, base, fdef.line, do_py_ops) + return gen_glue_property(builder, base_sig, target, cls, base, fdef.line, do_py_ops) else: - return gen_glue_method(builder, sig, target, cls, base, fdef.line, do_py_ops) + return gen_glue_method(builder, base_sig, target, cls, base, fdef.line, do_py_ops) class ArgInfo(NamedTuple): @@ -594,7 +594,7 @@ def get_args(builder: IRBuilder, rt_args: Sequence[RuntimeArg], line: int) -> Ar def gen_glue_method( builder: IRBuilder, - sig: FuncSignature, + base_sig: FuncSignature, target: FuncIR, cls: ClassIR, base: ClassIR, @@ -626,16 +626,25 @@ def f(builder: IRBuilder, x: object) -> int: ... If do_pycall is True, then make the call using the C API instead of a native call. """ + check_native_override(builder, base_sig, target.decl.sig, line) + builder.enter() - builder.ret_types[-1] = sig.ret_type + builder.ret_types[-1] = base_sig.ret_type - rt_args = list(sig.args) + rt_args = list(base_sig.args) if target.decl.kind == FUNC_NORMAL: - rt_args[0] = RuntimeArg(sig.args[0].name, RInstance(cls)) + rt_args[0] = RuntimeArg(base_sig.args[0].name, RInstance(cls)) arg_info = get_args(builder, rt_args, line) args, arg_kinds, arg_names = arg_info.args, arg_info.arg_kinds, arg_info.arg_names + bitmap_args = None + if base_sig.num_bitmap_args: + args = args[: -base_sig.num_bitmap_args] + arg_kinds = arg_kinds[: -base_sig.num_bitmap_args] + arg_names = arg_names[: -base_sig.num_bitmap_args] + bitmap_args = builder.builder.args[-base_sig.num_bitmap_args :] + # We can do a passthrough *args/**kwargs with a native call, but if the # args need to get distributed out to arguments, we just let python handle it if any(kind.is_star() for kind in arg_kinds) and any( @@ -655,11 +664,15 @@ def f(builder: IRBuilder, x: object) -> int: ... first, target.name, args[st:], line, arg_kinds[st:], arg_names[st:] ) else: - retval = builder.builder.call(target.decl, args, arg_kinds, arg_names, line) - retval = builder.coerce(retval, sig.ret_type, line) + retval = builder.builder.call( + target.decl, args, arg_kinds, arg_names, line, bitmap_args=bitmap_args + ) + retval = builder.coerce(retval, base_sig.ret_type, line) builder.add(Return(retval)) arg_regs, _, blocks, ret_type, _ = builder.leave() + if base_sig.num_bitmap_args: + rt_args = rt_args[: -base_sig.num_bitmap_args] return FuncIR( FuncDecl( target.name + "__" + base.name + "_glue", @@ -673,6 +686,35 @@ def f(builder: IRBuilder, x: object) -> int: ... ) +def check_native_override( + builder: IRBuilder, base_sig: FuncSignature, sub_sig: FuncSignature, line: int +) -> None: + """Report an error if an override changes signature in unsupported ways. + + Glue methods can work around many signature changes but not all of them. + """ + for base_arg, sub_arg in zip(base_sig.real_args(), sub_sig.real_args()): + if base_arg.type.error_overlap: + if not base_arg.optional and sub_arg.optional and base_sig.num_bitmap_args: + # This would change the meanings of bits in the argument defaults + # bitmap, which we don't support. We'd need to do tricky bit + # manipulations to support this generally. + builder.error( + "An argument with type " + + f'"{base_arg.type}" cannot be given a default value in a method override', + line, + ) + if base_arg.type.error_overlap or sub_arg.type.error_overlap: + if not is_same_type(base_arg.type, sub_arg.type): + # This would change from signaling a default via an error value to + # signaling a default via bitmap, which we don't support. + builder.error( + "Incompatible argument type " + + f'"{sub_arg.type}" (base class has type "{base_arg.type}")', + line, + ) + + def gen_glue_property( builder: IRBuilder, sig: FuncSignature, @@ -933,7 +975,7 @@ def maybe_insert_into_registry_dict(builder: IRBuilder, fitem: FuncDef) -> None: line = fitem.line is_singledispatch_main_func = fitem in builder.singledispatch_impls # dict of singledispatch_func to list of register_types (fitem is the function to register) - to_register: DefaultDict[FuncDef, list[TypeInfo]] = defaultdict(list) + to_register: defaultdict[FuncDef, list[TypeInfo]] = defaultdict(list) for main_func, impls in builder.singledispatch_impls.items(): for dispatch_type, impl in impls: if fitem == impl: diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index b0648d6e4c5d..fe0af5b13a73 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -18,9 +18,12 @@ from mypy.operators import op_methods from mypy.types import AnyType, TypeOfAny from mypyc.common import ( + BITMAP_BITS, FAST_ISINSTANCE_MAX_SUBCLASSES, MAX_LITERAL_SHORT_INT, + MAX_SHORT_INT, MIN_LITERAL_SHORT_INT, + MIN_SHORT_INT, PLATFORM_SIZE, use_method_vectorcall, use_vectorcall, @@ -42,6 +45,7 @@ CallC, Cast, ComparisonOp, + Extend, GetAttr, GetElementPtr, Goto, @@ -63,6 +67,7 @@ Unbox, Unreachable, Value, + int_op_to_id, ) from mypyc.ir.rtypes import ( PyListObject, @@ -71,13 +76,16 @@ PyVarObject, RArray, RInstance, + RPrimitive, RTuple, RType, RUnion, bit_rprimitive, + bitmap_rprimitive, bool_rprimitive, bytes_rprimitive, c_int_rprimitive, + c_pointer_rprimitive, c_pyssize_t_rprimitive, c_size_t_rprimitive, dict_rprimitive, @@ -87,6 +95,10 @@ is_bool_rprimitive, is_bytes_rprimitive, is_dict_rprimitive, + is_fixed_width_rtype, + is_int32_rprimitive, + is_int64_rprimitive, + is_int_rprimitive, is_list_rprimitive, is_none_rprimitive, is_set_rprimitive, @@ -124,7 +136,18 @@ py_vectorcall_method_op, py_vectorcall_op, ) -from mypyc.primitives.int_ops import int_comparison_op_mapping +from mypyc.primitives.int_ops import ( + int32_divide_op, + int32_mod_op, + int32_overflow, + int64_divide_op, + int64_mod_op, + int64_to_int_op, + int_comparison_op_mapping, + int_to_int32_op, + int_to_int64_op, + ssize_t_to_int_op, +) from mypyc.primitives.list_ops import list_build_op, list_extend_op, new_list_op from mypyc.primitives.misc_ops import bool_op, fast_isinstance_op, none_object_op from mypyc.primitives.registry import ( @@ -153,6 +176,29 @@ # From CPython PY_VECTORCALL_ARGUMENTS_OFFSET: Final = 1 << (PLATFORM_SIZE * 8 - 1) +FIXED_WIDTH_INT_BINARY_OPS: Final = { + "+", + "-", + "*", + "//", + "%", + "&", + "|", + "^", + "<<", + ">>", + "+=", + "-=", + "*=", + "//=", + "%=", + "&=", + "|=", + "^=", + "<<=", + ">>=", +} + class LowLevelIRBuilder: def __init__(self, current_module: str, mapper: Mapper, options: CompilerOptions) -> None: @@ -250,17 +296,43 @@ def coerce( Returns the register with the converted value (may be same as src). """ - if src.type.is_unboxed and not target_type.is_unboxed: + src_type = src.type + if src_type.is_unboxed and not target_type.is_unboxed: + # Unboxed -> boxed return self.box(src) - if (src.type.is_unboxed and target_type.is_unboxed) and not is_runtime_subtype( - src.type, target_type + if (src_type.is_unboxed and target_type.is_unboxed) and not is_runtime_subtype( + src_type, target_type ): - # To go from one unboxed type to another, we go through a boxed - # in-between value, for simplicity. - tmp = self.box(src) - return self.unbox_or_cast(tmp, target_type, line) - if (not src.type.is_unboxed and target_type.is_unboxed) or not is_subtype( - src.type, target_type + if ( + isinstance(src, Integer) + and is_short_int_rprimitive(src_type) + and is_fixed_width_rtype(target_type) + ): + # TODO: range check + return Integer(src.value >> 1, target_type) + elif is_int_rprimitive(src_type) and is_fixed_width_rtype(target_type): + return self.coerce_int_to_fixed_width(src, target_type, line) + elif is_fixed_width_rtype(src_type) and is_int_rprimitive(target_type): + return self.coerce_fixed_width_to_int(src, line) + elif is_short_int_rprimitive(src_type) and is_fixed_width_rtype(target_type): + return self.coerce_short_int_to_fixed_width(src, target_type, line) + elif ( + isinstance(src_type, RPrimitive) + and isinstance(target_type, RPrimitive) + and src_type.is_native_int + and target_type.is_native_int + and src_type.size == target_type.size + and src_type.is_signed == target_type.is_signed + ): + # Equivalent types + return src + else: + # To go from one unboxed type to another, we go through a boxed + # in-between value, for simplicity. + tmp = self.box(src) + return self.unbox_or_cast(tmp, target_type, line) + if (not src_type.is_unboxed and target_type.is_unboxed) or not is_subtype( + src_type, target_type ): return self.unbox_or_cast(src, target_type, line, can_borrow=can_borrow) elif force: @@ -269,6 +341,133 @@ def coerce( return tmp return src + def coerce_int_to_fixed_width(self, src: Value, target_type: RType, line: int) -> Value: + assert is_fixed_width_rtype(target_type), target_type + assert isinstance(target_type, RPrimitive) + + res = Register(target_type) + + fast, slow, end = BasicBlock(), BasicBlock(), BasicBlock() + + check = self.check_tagged_short_int(src, line) + self.add(Branch(check, fast, slow, Branch.BOOL)) + + self.activate_block(fast) + + size = target_type.size + if size < int_rprimitive.size: + # Add a range check when the target type is smaller than the source tyoe + fast2, fast3 = BasicBlock(), BasicBlock() + upper_bound = 1 << (size * 8 - 1) + check2 = self.add(ComparisonOp(src, Integer(upper_bound, src.type), ComparisonOp.SLT)) + self.add(Branch(check2, fast2, slow, Branch.BOOL)) + self.activate_block(fast2) + check3 = self.add(ComparisonOp(src, Integer(-upper_bound, src.type), ComparisonOp.SGE)) + self.add(Branch(check3, fast3, slow, Branch.BOOL)) + self.activate_block(fast3) + tmp = self.int_op( + c_pyssize_t_rprimitive, + src, + Integer(1, c_pyssize_t_rprimitive), + IntOp.RIGHT_SHIFT, + line, + ) + tmp = self.add(Truncate(tmp, target_type)) + else: + if size > int_rprimitive.size: + tmp = self.add(Extend(src, target_type, signed=True)) + else: + tmp = src + tmp = self.int_op(target_type, tmp, Integer(1, target_type), IntOp.RIGHT_SHIFT, line) + + self.add(Assign(res, tmp)) + self.goto(end) + + self.activate_block(slow) + if is_int64_rprimitive(target_type) or ( + is_int32_rprimitive(target_type) and size == int_rprimitive.size + ): + # Slow path calls a library function that handles more complex logic + ptr = self.int_op( + pointer_rprimitive, src, Integer(1, pointer_rprimitive), IntOp.XOR, line + ) + ptr2 = Register(c_pointer_rprimitive) + self.add(Assign(ptr2, ptr)) + if is_int64_rprimitive(target_type): + conv_op = int_to_int64_op + else: + conv_op = int_to_int32_op + tmp = self.call_c(conv_op, [ptr2], line) + self.add(Assign(res, tmp)) + self.add(KeepAlive([src])) + self.goto(end) + elif is_int32_rprimitive(target_type): + # Slow path just always generates an OverflowError + self.call_c(int32_overflow, [], line) + self.add(Unreachable()) + else: + assert False, target_type + + self.activate_block(end) + return res + + def coerce_short_int_to_fixed_width(self, src: Value, target_type: RType, line: int) -> Value: + if is_int64_rprimitive(target_type): + return self.int_op(target_type, src, Integer(1, target_type), IntOp.RIGHT_SHIFT, line) + # TODO: i32 + assert False, (src.type, target_type) + + def coerce_fixed_width_to_int(self, src: Value, line: int) -> Value: + if is_int32_rprimitive(src.type) and PLATFORM_SIZE == 8: + # Simple case -- just sign extend and shift. + extended = self.add(Extend(src, c_pyssize_t_rprimitive, signed=True)) + return self.int_op( + int_rprimitive, + extended, + Integer(1, c_pyssize_t_rprimitive), + IntOp.LEFT_SHIFT, + line, + ) + + assert is_fixed_width_rtype(src.type) + assert isinstance(src.type, RPrimitive) + src_type = src.type + + res = Register(int_rprimitive) + + fast, fast2, slow, end = BasicBlock(), BasicBlock(), BasicBlock(), BasicBlock() + + c1 = self.add(ComparisonOp(src, Integer(MAX_SHORT_INT, src_type), ComparisonOp.SLE)) + self.add(Branch(c1, fast, slow, Branch.BOOL)) + + self.activate_block(fast) + c2 = self.add(ComparisonOp(src, Integer(MIN_SHORT_INT, src_type), ComparisonOp.SGE)) + self.add(Branch(c2, fast2, slow, Branch.BOOL)) + + self.activate_block(slow) + if is_int64_rprimitive(src_type): + conv_op = int64_to_int_op + elif is_int32_rprimitive(src_type): + assert PLATFORM_SIZE == 4 + conv_op = ssize_t_to_int_op + else: + assert False, src_type + x = self.call_c(conv_op, [src], line) + self.add(Assign(res, x)) + self.goto(end) + + self.activate_block(fast2) + if int_rprimitive.size < src_type.size: + tmp = self.add(Truncate(src, c_pyssize_t_rprimitive)) + else: + tmp = src + s = self.int_op(int_rprimitive, tmp, Integer(1, tmp.type), IntOp.LEFT_SHIFT, line) + self.add(Assign(res, s)) + self.goto(end) + + self.activate_block(end) + return res + def coerce_nullable(self, src: Value, target_type: RType, line: int) -> Value: """Generate a coercion from a potentially null value.""" if src.type.is_unboxed == target_type.is_unboxed and ( @@ -305,9 +504,12 @@ def get_attr( and obj.type.class_ir.is_ext_class and obj.type.class_ir.has_attr(attr) ): - if borrow: + op = GetAttr(obj, attr, line, borrow=borrow) + # For non-refcounted attribute types, the borrow might be + # disabled even if requested, so don't check 'borrow'. + if op.is_borrowed: self.keep_alives.append(obj) - return self.add(GetAttr(obj, attr, line, borrow=borrow)) + return self.add(op) elif isinstance(obj.type, RUnion): return self.union_get_attr(obj, obj.type, attr, result_type, line) else: @@ -733,10 +935,19 @@ def call( arg_kinds: list[ArgKind], arg_names: Sequence[str | None], line: int, + *, + bitmap_args: list[Register] | None = None, ) -> Value: - """Call a native function.""" + """Call a native function. + + If bitmap_args is given, they override the values of (some) of the bitmap + arguments used to track the presence of values for certain arguments. By + default, the values of the bitmap arguments are inferred from args. + """ # Normalize args to positionals. - args = self.native_args_to_positional(args, arg_kinds, arg_names, decl.sig, line) + args = self.native_args_to_positional( + args, arg_kinds, arg_names, decl.sig, line, bitmap_args=bitmap_args + ) return self.add(Call(decl, args, line)) def native_args_to_positional( @@ -746,6 +957,8 @@ def native_args_to_positional( arg_names: Sequence[str | None], sig: FuncSignature, line: int, + *, + bitmap_args: list[Register] | None = None, ) -> list[Value]: """Prepare arguments for a native call. @@ -756,8 +969,14 @@ def native_args_to_positional( and coerce arguments to the appropriate type. """ - sig_arg_kinds = [arg.kind for arg in sig.args] - sig_arg_names = [arg.name for arg in sig.args] + sig_args = sig.args + n = sig.num_bitmap_args + if n: + sig_args = sig_args[:-n] + + sig_arg_kinds = [arg.kind for arg in sig_args] + sig_arg_names = [arg.name for arg in sig_args] + concrete_kinds = [concrete_arg_kind(arg_kind) for arg_kind in arg_kinds] formal_to_actual = map_actuals_to_formals( concrete_kinds, @@ -770,7 +989,7 @@ def native_args_to_positional( # First scan for */** and construct those has_star = has_star2 = False star_arg_entries = [] - for lst, arg in zip(formal_to_actual, sig.args): + for lst, arg in zip(formal_to_actual, sig_args): if arg.kind.is_star(): star_arg_entries.extend([(args[i], arg_kinds[i], arg_names[i]) for i in lst]) has_star = has_star or arg.kind == ARG_STAR @@ -783,8 +1002,8 @@ def native_args_to_positional( # Flatten out the arguments, loading error values for default # arguments, constructing tuples/dicts for star args, and # coercing everything to the expected type. - output_args = [] - for lst, arg in zip(formal_to_actual, sig.args): + output_args: list[Value] = [] + for lst, arg in zip(formal_to_actual, sig_args): if arg.kind == ARG_STAR: assert star_arg output_arg = star_arg @@ -792,7 +1011,10 @@ def native_args_to_positional( assert star2_arg output_arg = star2_arg elif not lst: - output_arg = self.add(LoadErrorValue(arg.type, is_borrowed=True)) + if is_fixed_width_rtype(arg.type): + output_arg = Integer(0, arg.type) + else: + output_arg = self.add(LoadErrorValue(arg.type, is_borrowed=True)) else: base_arg = args[lst[0]] @@ -803,6 +1025,22 @@ def native_args_to_positional( output_args.append(output_arg) + for i in reversed(range(n)): + if bitmap_args and i < len(bitmap_args): + # Use override provided by caller + output_args.append(bitmap_args[i]) + continue + # Infer values of bitmap args + bitmap = 0 + c = 0 + for lst, arg in zip(formal_to_actual, sig_args): + if arg.kind.is_optional() and is_fixed_width_rtype(arg.type): + if i * BITMAP_BITS <= c < (i + 1) * BITMAP_BITS: + if lst: + bitmap |= 1 << (c & (BITMAP_BITS - 1)) + c += 1 + output_args.append(Integer(bitmap, bitmap_rprimitive)) + return output_args def gen_method_call( @@ -965,7 +1203,13 @@ def load_native_type_object(self, fullname: str) -> Value: return self.add(LoadStatic(object_rprimitive, name, module, NAMESPACE_TYPE)) # Other primitive operations + def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: + """Perform a binary operation. + + Generate specialized operations based on operand types, with a fallback + to generic operations. + """ ltype = lreg.type rtype = rreg.type @@ -998,6 +1242,58 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: return self.bool_bitwise_op(lreg, rreg, op[0], line) if isinstance(rtype, RInstance) and op in ("in", "not in"): return self.translate_instance_contains(rreg, lreg, op, line) + if is_fixed_width_rtype(ltype): + if op in FIXED_WIDTH_INT_BINARY_OPS: + if op.endswith("="): + op = op[:-1] + if is_fixed_width_rtype(rtype) or is_tagged(rtype): + if op != "//": + op_id = int_op_to_id[op] + else: + op_id = IntOp.DIV + return self.fixed_width_int_op(ltype, lreg, rreg, op_id, line) + if isinstance(rreg, Integer): + # TODO: Check what kind of Integer + if op != "//": + op_id = int_op_to_id[op] + else: + op_id = IntOp.DIV + return self.fixed_width_int_op( + ltype, lreg, Integer(rreg.value >> 1, ltype), op_id, line + ) + elif op in ComparisonOp.signed_ops: + if is_int_rprimitive(rtype): + rreg = self.coerce_int_to_fixed_width(rreg, ltype, line) + op_id = ComparisonOp.signed_ops[op] + if is_fixed_width_rtype(rreg.type): + return self.comparison_op(lreg, rreg, op_id, line) + if isinstance(rreg, Integer): + return self.comparison_op(lreg, Integer(rreg.value >> 1, ltype), op_id, line) + elif is_fixed_width_rtype(rtype): + if ( + isinstance(lreg, Integer) or is_tagged(ltype) + ) and op in FIXED_WIDTH_INT_BINARY_OPS: + if op.endswith("="): + op = op[:-1] + # TODO: Support comparison ops (similar to above) + if op != "//": + op_id = int_op_to_id[op] + else: + op_id = IntOp.DIV + if isinstance(lreg, Integer): + # TODO: Check what kind of Integer + return self.fixed_width_int_op( + rtype, Integer(lreg.value >> 1, rtype), rreg, op_id, line + ) + else: + return self.fixed_width_int_op(rtype, lreg, rreg, op_id, line) + elif op in ComparisonOp.signed_ops: + if is_int_rprimitive(ltype): + lreg = self.coerce_int_to_fixed_width(lreg, rtype, line) + op_id = ComparisonOp.signed_ops[op] + if isinstance(lreg, Integer): + return self.comparison_op(Integer(lreg.value >> 1, rtype), rreg, op_id, line) + return self.comparison_op(lreg, rreg, op_id, line) call_c_ops_candidates = binary_ops.get(op, []) target = self.matching_call_c(call_c_ops_candidates, [lreg, rreg], line) @@ -1210,9 +1506,26 @@ def unary_op(self, value: Value, expr_op: str, line: int) -> Value: typ = value.type if (is_bool_rprimitive(typ) or is_bit_rprimitive(typ)) and expr_op == "not": return self.unary_not(value, line) + if is_fixed_width_rtype(typ): + if expr_op == "-": + # Translate to '0 - x' + return self.int_op(typ, Integer(0, typ), value, IntOp.SUB, line) + elif expr_op == "~": + # Translate to 'x ^ -1' + return self.int_op(typ, value, Integer(-1, typ), IntOp.XOR, line) + elif expr_op == "+": + return value + if isinstance(value, Integer): + # TODO: Overflow? Unsigned? + num = value.value + if is_short_int_rprimitive(typ): + num >>= 1 + return Integer(-num, typ, value.line) if isinstance(typ, RInstance): if expr_op == "-": method = "__neg__" + elif expr_op == "+": + method = "__pos__" elif expr_op == "~": method = "__invert__" else: @@ -1334,6 +1647,9 @@ def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) -> zero = Integer(0, short_int_rprimitive) self.compare_tagged_condition(value, zero, "!=", true, false, value.line) return + elif is_fixed_width_rtype(value.type): + zero = Integer(0, value.type) + value = self.add(ComparisonOp(value, zero, ComparisonOp.NEQ)) elif is_same_type(value.type, str_rprimitive): value = self.call_c(str_check_if_true, [value], value.line) elif is_same_type(value.type, list_rprimitive) or is_same_type( @@ -1480,8 +1796,98 @@ def matching_call_c( return None def int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int) -> Value: + """Generate a native integer binary op. + + Use native/C semantics, which sometimes differ from Python + semantics. + + Args: + type: Either int64_rprimitive or int32_rprimitive + op: IntOp.* constant (e.g. IntOp.ADD) + """ return self.add(IntOp(type, lhs, rhs, op, line)) + def fixed_width_int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int) -> Value: + """Generate a binary op using Python fixed-width integer semantics. + + These may differ in overflow/rounding behavior from native/C ops. + + Args: + type: Either int64_rprimitive or int32_rprimitive + op: IntOp.* constant (e.g. IntOp.ADD) + """ + lhs = self.coerce(lhs, type, line) + rhs = self.coerce(rhs, type, line) + if op == IntOp.DIV: + # Inline simple division by a constant, so that C + # compilers can optimize more + if isinstance(rhs, Integer) and rhs.value not in (-1, 0): + return self.inline_fixed_width_divide(type, lhs, rhs, line) + if is_int64_rprimitive(type): + prim = int64_divide_op + elif is_int32_rprimitive(type): + prim = int32_divide_op + else: + assert False, type + return self.call_c(prim, [lhs, rhs], line) + if op == IntOp.MOD: + # Inline simple % by a constant, so that C + # compilers can optimize more + if isinstance(rhs, Integer) and rhs.value not in (-1, 0): + return self.inline_fixed_width_mod(type, lhs, rhs, line) + if is_int64_rprimitive(type): + prim = int64_mod_op + elif is_int32_rprimitive(type): + prim = int32_mod_op + else: + assert False, type + return self.call_c(prim, [lhs, rhs], line) + return self.int_op(type, lhs, rhs, op, line) + + def inline_fixed_width_divide(self, type: RType, lhs: Value, rhs: Value, line: int) -> Value: + # Perform floor division (native division truncates) + res = Register(type) + div = self.int_op(type, lhs, rhs, IntOp.DIV, line) + self.add(Assign(res, div)) + diff_signs = self.is_different_native_int_signs(type, lhs, rhs, line) + tricky, adjust, done = BasicBlock(), BasicBlock(), BasicBlock() + self.add(Branch(diff_signs, done, tricky, Branch.BOOL)) + self.activate_block(tricky) + mul = self.int_op(type, res, rhs, IntOp.MUL, line) + mul_eq = self.add(ComparisonOp(mul, lhs, ComparisonOp.EQ, line)) + adjust = BasicBlock() + self.add(Branch(mul_eq, done, adjust, Branch.BOOL)) + self.activate_block(adjust) + adj = self.int_op(type, res, Integer(1, type), IntOp.SUB, line) + self.add(Assign(res, adj)) + self.add(Goto(done)) + self.activate_block(done) + return res + + def inline_fixed_width_mod(self, type: RType, lhs: Value, rhs: Value, line: int) -> Value: + # Perform floor modulus + res = Register(type) + mod = self.int_op(type, lhs, rhs, IntOp.MOD, line) + self.add(Assign(res, mod)) + diff_signs = self.is_different_native_int_signs(type, lhs, rhs, line) + tricky, adjust, done = BasicBlock(), BasicBlock(), BasicBlock() + self.add(Branch(diff_signs, done, tricky, Branch.BOOL)) + self.activate_block(tricky) + is_zero = self.add(ComparisonOp(res, Integer(0, type), ComparisonOp.EQ, line)) + adjust = BasicBlock() + self.add(Branch(is_zero, done, adjust, Branch.BOOL)) + self.activate_block(adjust) + adj = self.int_op(type, res, rhs, IntOp.ADD, line) + self.add(Assign(res, adj)) + self.add(Goto(done)) + self.activate_block(done) + return res + + def is_different_native_int_signs(self, type: RType, a: Value, b: Value, line: int) -> Value: + neg1 = self.add(ComparisonOp(a, Integer(0, type), ComparisonOp.SLT, line)) + neg2 = self.add(ComparisonOp(b, Integer(0, type), ComparisonOp.SLT, line)) + return self.add(ComparisonOp(neg1, neg2, ComparisonOp.EQ, line)) + def comparison_op(self, lhs: Value, rhs: Value, op: int, line: int) -> Value: return self.add(ComparisonOp(lhs, rhs, op, line)) diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index e20872979b7a..9bbb90aad207 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -57,7 +57,11 @@ def build_ir( options: CompilerOptions, errors: Errors, ) -> ModuleIRs: - """Build IR for a set of modules that have been type-checked by mypy.""" + """Build basic IR for a set of modules that have been type-checked by mypy. + + The returned IR is not complete and requires additional + transformations, such as the insertion of refcount handling. + """ build_type_map(mapper, modules, graph, types, options, errors) singledispatch_info = find_singledispatch_register_impls(modules, errors) diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 6d6ce1576b54..4364b2b6c511 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -32,6 +32,8 @@ bytes_rprimitive, dict_rprimitive, float_rprimitive, + int32_rprimitive, + int64_rprimitive, int_rprimitive, list_rprimitive, none_rprimitive, @@ -96,6 +98,10 @@ def type_to_rtype(self, typ: Type | None) -> RType: return RUnion([inst, object_rprimitive]) else: return inst + elif typ.type.fullname == "mypy_extensions.i64": + return int64_rprimitive + elif typ.type.fullname == "mypy_extensions.i32": + return int32_rprimitive else: return object_rprimitive elif isinstance(typ, TupleType): diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index e40dfa0d7c02..82162d1d0d0e 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -14,7 +14,7 @@ from __future__ import annotations from collections import defaultdict -from typing import DefaultDict, Iterable, NamedTuple, Tuple +from typing import Iterable, NamedTuple, Tuple from mypy.build import Graph from mypy.nodes import ( @@ -288,7 +288,8 @@ def prepare_class_def( init_sig.ret_type, ) - ctor_sig = FuncSignature(init_sig.args[1:], RInstance(ir)) + last_arg = len(init_sig.args) - init_sig.num_bitmap_args + ctor_sig = FuncSignature(init_sig.args[1:last_arg], RInstance(ir)) ir.ctor = FuncDecl(cdef.name, None, module_name, ctor_sig) mapper.func_to_decl[cdef.info] = ir.ctor @@ -379,7 +380,7 @@ def __init__(self, errors: Errors) -> None: super().__init__() # Map of main singledispatch function to list of registered implementations - self.singledispatch_impls: DefaultDict[FuncDef, list[RegisterImplInfo]] = defaultdict(list) + self.singledispatch_impls: defaultdict[FuncDef, list[RegisterImplInfo]] = defaultdict(list) # Map of decorated function to the indices of any decorators to remove self.decorators_to_remove: dict[FuncDef, list[int]] = {} diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index d09d1bd05687..5810482cd43d 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -32,14 +32,29 @@ TupleExpr, ) from mypy.types import AnyType, TypeOfAny -from mypyc.ir.ops import BasicBlock, Integer, RaiseStandardError, Register, Unreachable, Value +from mypyc.ir.ops import ( + BasicBlock, + Extend, + Integer, + RaiseStandardError, + Register, + Truncate, + Unreachable, + Value, +) from mypyc.ir.rtypes import ( + RInstance, RTuple, RType, bool_rprimitive, c_int_rprimitive, dict_rprimitive, + int32_rprimitive, + int64_rprimitive, is_dict_rprimitive, + is_int32_rprimitive, + is_int64_rprimitive, + is_int_rprimitive, is_list_rprimitive, list_rprimitive, set_rprimitive, @@ -138,6 +153,19 @@ def translate_globals(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Va return None +@specialize_function("builtins.abs") +def translate_abs(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + """Specialize calls on native classes that implement __abs__.""" + if len(expr.args) == 1 and expr.arg_kinds == [ARG_POS]: + arg = expr.args[0] + arg_typ = builder.node_type(arg) + if isinstance(arg_typ, RInstance) and arg_typ.class_ir.has_method("__abs__"): + obj = builder.accept(arg) + return builder.gen_method_call(obj, "__abs__", [], None, expr.line) + + return None + + @specialize_function("builtins.len") def translate_len(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) == 1 and expr.arg_kinds == [ARG_POS]: @@ -626,3 +654,37 @@ def translate_fstring(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Va return join_formatted_strings(builder, None, substitutions, expr.line) return None + + +@specialize_function("mypy_extensions.i64") +def translate_i64(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + arg_type = builder.node_type(arg) + if is_int64_rprimitive(arg_type): + return builder.accept(arg) + elif is_int32_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Extend(val, int64_rprimitive, signed=True, line=expr.line)) + elif is_int_rprimitive(arg_type): + val = builder.accept(arg) + return builder.coerce(val, int64_rprimitive, expr.line) + return None + + +@specialize_function("mypy_extensions.i32") +def translate_i32(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: + if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS: + return None + arg = expr.args[0] + arg_type = builder.node_type(arg) + if is_int32_rprimitive(arg_type): + return builder.accept(arg) + elif is_int64_rprimitive(arg_type): + val = builder.accept(arg) + return builder.add(Truncate(val, int32_rprimitive, line=expr.line)) + elif is_int_rprimitive(arg_type): + val = builder.accept(arg) + return builder.coerce(val, int32_rprimitive, expr.line) + return None diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 371a305e67b9..a1d36c011aa1 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -616,6 +616,8 @@ def transform_try_stmt(builder: IRBuilder, t: TryStmt) -> None: # constructs that we compile separately. When we have a # try/except/else/finally, we treat the try/except/else as the # body of a try/finally block. + if t.is_star: + builder.error("Exception groups and except* cannot be compiled yet", t.line) if t.finally_body: def transform_try_body() -> None: diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index 21d4d4cc5620..5ea2f65d5776 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -544,7 +544,7 @@ int64_t CPyInt64_Divide(int64_t x, int64_t y) { PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); return CPY_LL_INT_ERROR; } - if (y == -1 && x == -1LL << 63) { + if (y == -1 && x == INT64_MIN) { PyErr_SetString(PyExc_OverflowError, "integer division overflow"); return CPY_LL_INT_ERROR; } @@ -562,7 +562,7 @@ int64_t CPyInt64_Remainder(int64_t x, int64_t y) { return CPY_LL_INT_ERROR; } // Edge case: avoid core dump - if (y == -1 && x == -1LL << 63) { + if (y == -1 && x == INT64_MIN) { return 0; } int64_t d = x % y; @@ -607,7 +607,7 @@ int32_t CPyInt32_Divide(int32_t x, int32_t y) { PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); return CPY_LL_INT_ERROR; } - if (y == -1 && x == -1LL << 31) { + if (y == -1 && x == INT32_MIN) { PyErr_SetString(PyExc_OverflowError, "integer division overflow"); return CPY_LL_INT_ERROR; } @@ -625,7 +625,7 @@ int32_t CPyInt32_Remainder(int32_t x, int32_t y) { return CPY_LL_INT_ERROR; } // Edge case: avoid core dump - if (y == -1 && x == -1LL << 31) { + if (y == -1 && x == INT32_MIN) { return 0; } int32_t d = x % y; diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 90292ce61073..25f33c5f56c7 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -3,6 +3,7 @@ // These are registered in mypyc.primitives.misc_ops. #include +#include #include "CPy.h" PyObject *CPy_GetCoro(PyObject *obj) @@ -285,6 +286,11 @@ PyObject *CPyType_FromTemplate(PyObject *template, Py_XDECREF(dummy_class); +#if PY_MINOR_VERSION == 11 + // This is a hack. Python 3.11 doesn't include good public APIs to work with managed + // dicts, which are the default for heap types. So we try to opt-out until Python 3.12. + t->ht_type.tp_flags &= ~Py_TPFLAGS_MANAGED_DICT; +#endif return (PyObject *)t; error: diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index cdaa94931604..f6817ad024b7 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -145,6 +145,16 @@ priority=0, ) +# abs(obj) +function_op( + name="builtins.abs", + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="PyNumber_Absolute", + error_kind=ERR_MAGIC, + priority=0, +) + # obj1[obj2] method_op( name="__getitem__", diff --git a/mypyc/sametype.py b/mypyc/sametype.py index a3cfd5c08059..056ed683e5b8 100644 --- a/mypyc/sametype.py +++ b/mypyc/sametype.py @@ -35,7 +35,7 @@ def is_same_method_signature(a: FuncSignature, b: FuncSignature) -> bool: len(a.args) == len(b.args) and is_same_type(a.ret_type, b.ret_type) and all( - is_same_type(t1.type, t2.type) and t1.name == t2.name + is_same_type(t1.type, t2.type) and t1.name == t2.name and t1.optional == t2.optional for t1, t2 in zip(a.args[1:], b.args[1:]) ) ) diff --git a/mypyc/test-data/commandline.test b/mypyc/test-data/commandline.test index cfd0d708bbda..6612df9e1886 100644 --- a/mypyc/test-data/commandline.test +++ b/mypyc/test-data/commandline.test @@ -171,7 +171,7 @@ class Nope(Trait1, Concrete2): # E: Non-trait bases must appear first in parent class NonExt2: @property # E: Property setters not supported in non-extension classes def test(self) -> int: - pass + return 0 @test.setter def test(self, x: int) -> None: diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index 8b186e234c5e..187551249676 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -514,3 +514,128 @@ L13: L14: dec_ref r9 goto L8 + +[case testExceptionWithOverlappingErrorValue] +from mypy_extensions import i64 + +def f() -> i64: + return 0 + +def g() -> i64: + return f() +[out] +def f(): +L0: + return 0 +def g(): + r0 :: int64 + r1 :: bit + r2 :: object + r3 :: int64 +L0: + r0 = f() + r1 = r0 == -113 + if r1 goto L2 else goto L1 :: bool +L1: + return r0 +L2: + r2 = PyErr_Occurred() + if not is_error(r2) goto L3 (error at g:7) else goto L1 +L3: + r3 = :: int64 + return r3 + +[case testExceptionWithNativeAttributeGetAndSet] +class C: + def __init__(self, x: int) -> None: + self.x = x + +def foo(c: C, x: int) -> None: + c.x = x - c.x +[out] +def C.__init__(self, x): + self :: __main__.C + x :: int +L0: + inc_ref x :: int + self.x = x + return 1 +def foo(c, x): + c :: __main__.C + x, r0, r1 :: int + r2 :: bool +L0: + r0 = borrow c.x + r1 = CPyTagged_Subtract(x, r0) + c.x = r1 + return 1 + +[case testExceptionWithLowLevelIntAttribute] +from mypy_extensions import i32, i64 + +class C: + def __init__(self, x: i32, y: i64) -> None: + self.x = x + self.y = y + +def f(c: C) -> None: + c.x + c.y +[out] +def C.__init__(self, x, y): + self :: __main__.C + x :: int32 + y :: int64 +L0: + self.x = x + self.y = y + return 1 +def f(c): + c :: __main__.C + r0 :: int32 + r1 :: int64 +L0: + r0 = c.x + r1 = c.y + return 1 + +[case testConditionallyUndefinedI64] +from mypy_extensions import i64 + +def f(x: i64) -> i64: + if x: + y: i64 = 2 + return y +[out] +def f(x): + x, r0, y :: int64 + __locals_bitmap0 :: uint32 + r1 :: bit + r2, r3 :: uint32 + r4 :: bit + r5 :: bool + r6 :: int64 +L0: + r0 = :: int64 + y = r0 + __locals_bitmap0 = 0 + r1 = x != 0 + if r1 goto L1 else goto L2 :: bool +L1: + y = 2 + r2 = __locals_bitmap0 | 1 + __locals_bitmap0 = r2 +L2: + r3 = __locals_bitmap0 & 1 + r4 = r3 == 0 + if r4 goto L3 else goto L5 :: bool +L3: + r5 = raise UnboundLocalError('local variable "y" referenced before assignment') + if not r5 goto L6 (error at f:-1) else goto L4 :: bool +L4: + unreachable +L5: + return y +L6: + r6 = :: int64 + return r6 diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index d8c4333cafad..0e437f4597ea 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -3,7 +3,7 @@ from typing import ( TypeVar, Generic, List, Iterator, Iterable, Dict, Optional, Tuple, Any, Set, - overload, Mapping, Union, Callable, Sequence, FrozenSet + overload, Mapping, Union, Callable, Sequence, FrozenSet, Protocol ) T = TypeVar('T') @@ -12,6 +12,10 @@ K = TypeVar('K') # for keys in mapping V = TypeVar('V') # for values in mapping +class __SupportsAbs(Protocol[T_co]): + def __abs__(self) -> T_co: pass + + class object: def __init__(self) -> None: pass def __eq__(self, x: object) -> bool: pass @@ -40,6 +44,7 @@ def __truediv__(self, x: float) -> float: pass def __mod__(self, x: int) -> int: pass def __neg__(self) -> int: pass def __pos__(self) -> int: pass + def __abs__(self) -> int: pass def __invert__(self) -> int: pass def __and__(self, n: int) -> int: pass def __or__(self, n: int) -> int: pass @@ -88,6 +93,9 @@ def __sub__(self, n: float) -> float: pass def __mul__(self, n: float) -> float: pass def __truediv__(self, n: float) -> float: pass def __neg__(self) -> float: pass + def __pos__(self) -> float: pass + def __abs__(self) -> float: pass + def __invert__(self) -> float: pass class complex: def __init__(self, x: object, y: object = None) -> None: pass @@ -164,6 +172,7 @@ def __rmul__(self, i: int) -> List[T]: pass def __iter__(self) -> Iterator[T]: pass def __len__(self) -> int: pass def __contains__(self, item: object) -> int: ... + def __add__(self, x: List[T]) -> List[T]: ... def append(self, x: T) -> None: pass def pop(self, i: int = -1) -> T: pass def count(self, T) -> int: pass @@ -248,39 +257,26 @@ class Exception(BaseException): def __init__(self, message: Optional[str] = None) -> None: pass class Warning(Exception): pass - class UserWarning(Warning): pass - class TypeError(Exception): pass - class ValueError(Exception): pass - class AttributeError(Exception): pass - class ImportError(Exception): pass - class NameError(Exception): pass - class LookupError(Exception): pass - class KeyError(LookupError): pass - class IndexError(LookupError): pass - class RuntimeError(Exception): pass - class UnicodeEncodeError(RuntimeError): pass - class UnicodeDecodeError(RuntimeError): pass - class NotImplementedError(RuntimeError): pass class StopIteration(Exception): value: Any class ArithmeticError(Exception): pass - -class ZeroDivisionError(Exception): pass +class ZeroDivisionError(ArithmeticError): pass +class OverflowError(ArithmeticError): pass class GeneratorExit(BaseException): pass @@ -308,7 +304,7 @@ def zip(x: Iterable[T], y: Iterable[S]) -> Iterator[Tuple[T, S]]: ... @overload def zip(x: Iterable[T], y: Iterable[S], z: Iterable[V]) -> Iterator[Tuple[T, S, V]]: ... def eval(e: str) -> Any: ... -def abs(x: float) -> float: ... +def abs(x: __SupportsAbs[T]) -> T: ... def exit() -> None: ... def min(x: T, y: T) -> T: ... def max(x: T, y: T) -> T: ... diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 0080b1b4f223..7b4fcc9fc1ca 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -12,7 +12,7 @@ def assertRaises(typ: type, msg: str = '') -> Iterator[None]: try: yield except Exception as e: - assert isinstance(e, typ), f"{e} is not a {typ.__name__}" + assert isinstance(e, typ), f"{e!r} is not a {typ.__name__}" assert msg in str(e), f'Message "{e}" does not match "{msg}"' else: assert False, f"Expected {typ.__name__} but got no exception" diff --git a/mypyc/test-data/irbuild-any.test b/mypyc/test-data/irbuild-any.test index bace026bc957..bcf9a1880635 100644 --- a/mypyc/test-data/irbuild-any.test +++ b/mypyc/test-data/irbuild-any.test @@ -176,3 +176,25 @@ L6: r4 = unbox(int, r3) n = r4 return 1 + +[case testAbsSpecialization] +# Specialization of native classes that implement __abs__ is checked in +# irbuild-dunders.test +def f() -> None: + a = abs(1) + b = abs(1.1) +[out] +def f(): + r0, r1 :: object + r2, a :: int + r3, r4, b :: float +L0: + r0 = object 1 + r1 = PyNumber_Absolute(r0) + r2 = unbox(int, r1) + a = r2 + r3 = 1.1 + r4 = PyNumber_Absolute(r3) + b = r4 + return 1 + diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index 8e54b25b673b..4f5c9487bb1d 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -2228,243 +2228,6 @@ L0: r1 = CPyTagged_Multiply(4, r0) return r1 -[case testPropertyDerivedGen] -from typing import Callable -class BaseProperty: - @property - def value(self) -> object: - return self._incrementer - - @property - def bad_value(self) -> object: - return self._incrementer - - @property - def next(self) -> BaseProperty: - return BaseProperty(self._incrementer + 1) - - def __init__(self, value: int) -> None: - self._incrementer = value - -class DerivedProperty(BaseProperty): - @property - def value(self) -> int: - return self._incrementer - - @property - def bad_value(self) -> object: - return self._incrementer - - @property - def next(self) -> DerivedProperty: - return DerivedProperty(self._incr_func, self._incr_func(self.value)) - - def __init__(self, incr_func: Callable[[int], int], value: int) -> None: - BaseProperty.__init__(self, value) - self._incr_func = incr_func - - -class AgainProperty(DerivedProperty): - @property - def next(self) -> AgainProperty: - return AgainProperty(self._incr_func, self._incr_func(self._incr_func(self.value))) - - @property - def bad_value(self) -> int: - return self._incrementer -[out] -def BaseProperty.value(self): - self :: __main__.BaseProperty - r0 :: int - r1 :: object -L0: - r0 = self._incrementer - r1 = box(int, r0) - return r1 -def BaseProperty.bad_value(self): - self :: __main__.BaseProperty - r0 :: int - r1 :: object -L0: - r0 = self._incrementer - r1 = box(int, r0) - return r1 -def BaseProperty.next(self): - self :: __main__.BaseProperty - r0, r1 :: int - r2 :: __main__.BaseProperty -L0: - r0 = borrow self._incrementer - r1 = CPyTagged_Add(r0, 2) - keep_alive self - r2 = BaseProperty(r1) - return r2 -def BaseProperty.__init__(self, value): - self :: __main__.BaseProperty - value :: int -L0: - self._incrementer = value - return 1 -def DerivedProperty.value(self): - self :: __main__.DerivedProperty - r0 :: int -L0: - r0 = self._incrementer - return r0 -def DerivedProperty.value__BaseProperty_glue(__mypyc_self__): - __mypyc_self__ :: __main__.DerivedProperty - r0 :: int - r1 :: object -L0: - r0 = __mypyc_self__.value - r1 = box(int, r0) - return r1 -def DerivedProperty.bad_value(self): - self :: __main__.DerivedProperty - r0 :: int - r1 :: object -L0: - r0 = self._incrementer - r1 = box(int, r0) - return r1 -def DerivedProperty.next(self): - self :: __main__.DerivedProperty - r0 :: object - r1 :: int - r2, r3, r4 :: object - r5 :: int - r6 :: __main__.DerivedProperty -L0: - r0 = self._incr_func - r1 = self.value - r2 = self._incr_func - r3 = box(int, r1) - r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) - r5 = unbox(int, r4) - r6 = DerivedProperty(r0, r5) - return r6 -def DerivedProperty.next__BaseProperty_glue(__mypyc_self__): - __mypyc_self__, r0 :: __main__.DerivedProperty -L0: - r0 = __mypyc_self__.next - return r0 -def DerivedProperty.__init__(self, incr_func, value): - self :: __main__.DerivedProperty - incr_func :: object - value :: int - r0 :: None -L0: - r0 = BaseProperty.__init__(self, value) - self._incr_func = incr_func - return 1 -def AgainProperty.next(self): - self :: __main__.AgainProperty - r0 :: object - r1 :: int - r2, r3, r4 :: object - r5 :: int - r6, r7, r8 :: object - r9 :: int - r10 :: __main__.AgainProperty -L0: - r0 = self._incr_func - r1 = self.value - r2 = self._incr_func - r3 = box(int, r1) - r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) - r5 = unbox(int, r4) - r6 = self._incr_func - r7 = box(int, r5) - r8 = PyObject_CallFunctionObjArgs(r6, r7, 0) - r9 = unbox(int, r8) - r10 = AgainProperty(r0, r9) - return r10 -def AgainProperty.next__DerivedProperty_glue(__mypyc_self__): - __mypyc_self__, r0 :: __main__.AgainProperty -L0: - r0 = __mypyc_self__.next - return r0 -def AgainProperty.next__BaseProperty_glue(__mypyc_self__): - __mypyc_self__, r0 :: __main__.AgainProperty -L0: - r0 = __mypyc_self__.next - return r0 -def AgainProperty.bad_value(self): - self :: __main__.AgainProperty - r0 :: int -L0: - r0 = self._incrementer - return r0 -def AgainProperty.bad_value__DerivedProperty_glue(__mypyc_self__): - __mypyc_self__ :: __main__.AgainProperty - r0 :: int - r1 :: object -L0: - r0 = __mypyc_self__.bad_value - r1 = box(int, r0) - return r1 -def AgainProperty.bad_value__BaseProperty_glue(__mypyc_self__): - __mypyc_self__ :: __main__.AgainProperty - r0 :: int - r1 :: object -L0: - r0 = __mypyc_self__.bad_value - r1 = box(int, r0) - return r1 - -[case testPropertyTraitSubclassing] -from mypy_extensions import trait -@trait -class SubclassedTrait: - @property - def this(self) -> SubclassedTrait: - return self - - @property - def boxed(self) -> object: - return 3 - -class DerivingObject(SubclassedTrait): - @property - def this(self) -> DerivingObject: - return self - - @property - def boxed(self) -> int: - return 5 -[out] -def SubclassedTrait.this(self): - self :: __main__.SubclassedTrait -L0: - return self -def SubclassedTrait.boxed(self): - self :: __main__.SubclassedTrait - r0 :: object -L0: - r0 = object 3 - return r0 -def DerivingObject.this(self): - self :: __main__.DerivingObject -L0: - return self -def DerivingObject.this__SubclassedTrait_glue(__mypyc_self__): - __mypyc_self__, r0 :: __main__.DerivingObject -L0: - r0 = __mypyc_self__.this - return r0 -def DerivingObject.boxed(self): - self :: __main__.DerivingObject -L0: - return 10 -def DerivingObject.boxed__SubclassedTrait_glue(__mypyc_self__): - __mypyc_self__ :: __main__.DerivingObject - r0 :: int - r1 :: object -L0: - r0 = __mypyc_self__.boxed - r1 = box(int, r0) - return r1 - [case testNativeIndex] from typing import List class A: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 5a574ac44354..700a529f9627 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -181,94 +181,6 @@ L0: o.x = r1; r2 = is_error return o -[case testSubclassSpecialize2] -class A: - def foo(self, x: int) -> object: - return str(x) -class B(A): - def foo(self, x: object) -> object: - return x -class C(B): - def foo(self, x: object) -> int: - return id(x) - -def use_a(x: A, y: int) -> object: - return x.foo(y) - -def use_b(x: B, y: object) -> object: - return x.foo(y) - -def use_c(x: C, y: object) -> int: - return x.foo(y) -[out] -def A.foo(self, x): - self :: __main__.A - x :: int - r0 :: str -L0: - r0 = CPyTagged_Str(x) - return r0 -def B.foo(self, x): - self :: __main__.B - x :: object -L0: - return x -def B.foo__A_glue(self, x): - self :: __main__.B - x :: int - r0, r1 :: object -L0: - r0 = box(int, x) - r1 = B.foo(self, r0) - return r1 -def C.foo(self, x): - self :: __main__.C - x :: object - r0 :: int -L0: - r0 = CPyTagged_Id(x) - return r0 -def C.foo__B_glue(self, x): - self :: __main__.C - x :: object - r0 :: int - r1 :: object -L0: - r0 = C.foo(self, x) - r1 = box(int, r0) - return r1 -def C.foo__A_glue(self, x): - self :: __main__.C - x :: int - r0 :: object - r1 :: int - r2 :: object -L0: - r0 = box(int, x) - r1 = C.foo(self, r0) - r2 = box(int, r1) - return r2 -def use_a(x, y): - x :: __main__.A - y :: int - r0 :: object -L0: - r0 = x.foo(y) - return r0 -def use_b(x, y): - x :: __main__.B - y, r0 :: object -L0: - r0 = x.foo(y) - return r0 -def use_c(x, y): - x :: __main__.C - y :: object - r0 :: int -L0: - r0 = x.foo(y) - return r0 - [case testSubclass_toplevel] from typing import TypeVar, Generic from mypy_extensions import trait diff --git a/mypyc/test-data/irbuild-dunders.test b/mypyc/test-data/irbuild-dunders.test index d06a570aa7b0..24e708913354 100644 --- a/mypyc/test-data/irbuild-dunders.test +++ b/mypyc/test-data/irbuild-dunders.test @@ -148,11 +148,19 @@ class C: def __float__(self) -> float: return 4.0 + def __pos__(self) -> int: + return 5 + + def __abs__(self) -> int: + return 6 + def f(c: C) -> None: -c ~c int(c) float(c) + +c + abs(c) [out] def C.__neg__(self): self :: __main__.C @@ -172,10 +180,19 @@ def C.__float__(self): L0: r0 = 4.0 return r0 +def C.__pos__(self): + self :: __main__.C +L0: + return 10 +def C.__abs__(self): + self :: __main__.C +L0: + return 12 def f(c): c :: __main__.C r0, r1 :: int r2, r3, r4, r5 :: object + r6, r7 :: int L0: r0 = c.__neg__() r1 = c.__invert__() @@ -183,5 +200,7 @@ L0: r3 = PyObject_CallFunctionObjArgs(r2, c, 0) r4 = load_address PyFloat_Type r5 = PyObject_CallFunctionObjArgs(r4, c, 0) + r6 = c.__pos__() + r7 = c.__abs__() return 1 diff --git a/mypyc/test-data/irbuild-glue-methods.test b/mypyc/test-data/irbuild-glue-methods.test new file mode 100644 index 000000000000..6d749bf5dd84 --- /dev/null +++ b/mypyc/test-data/irbuild-glue-methods.test @@ -0,0 +1,437 @@ +# Test cases for glue methods. +# +# These are used when subclass method signature has a different representation +# compared to the base class. + +[case testSubclassSpecialize2] +class A: + def foo(self, x: int) -> object: + return str(x) +class B(A): + def foo(self, x: object) -> object: + return x +class C(B): + def foo(self, x: object) -> int: + return id(x) + +def use_a(x: A, y: int) -> object: + return x.foo(y) + +def use_b(x: B, y: object) -> object: + return x.foo(y) + +def use_c(x: C, y: object) -> int: + return x.foo(y) +[out] +def A.foo(self, x): + self :: __main__.A + x :: int + r0 :: str +L0: + r0 = CPyTagged_Str(x) + return r0 +def B.foo(self, x): + self :: __main__.B + x :: object +L0: + return x +def B.foo__A_glue(self, x): + self :: __main__.B + x :: int + r0, r1 :: object +L0: + r0 = box(int, x) + r1 = B.foo(self, r0) + return r1 +def C.foo(self, x): + self :: __main__.C + x :: object + r0 :: int +L0: + r0 = CPyTagged_Id(x) + return r0 +def C.foo__B_glue(self, x): + self :: __main__.C + x :: object + r0 :: int + r1 :: object +L0: + r0 = C.foo(self, x) + r1 = box(int, r0) + return r1 +def C.foo__A_glue(self, x): + self :: __main__.C + x :: int + r0 :: object + r1 :: int + r2 :: object +L0: + r0 = box(int, x) + r1 = C.foo(self, r0) + r2 = box(int, r1) + return r2 +def use_a(x, y): + x :: __main__.A + y :: int + r0 :: object +L0: + r0 = x.foo(y) + return r0 +def use_b(x, y): + x :: __main__.B + y, r0 :: object +L0: + r0 = x.foo(y) + return r0 +def use_c(x, y): + x :: __main__.C + y :: object + r0 :: int +L0: + r0 = x.foo(y) + return r0 + +[case testPropertyDerivedGen] +from typing import Callable +class BaseProperty: + @property + def value(self) -> object: + return self._incrementer + + @property + def bad_value(self) -> object: + return self._incrementer + + @property + def next(self) -> BaseProperty: + return BaseProperty(self._incrementer + 1) + + def __init__(self, value: int) -> None: + self._incrementer = value + +class DerivedProperty(BaseProperty): + @property + def value(self) -> int: + return self._incrementer + + @property + def bad_value(self) -> object: + return self._incrementer + + @property + def next(self) -> DerivedProperty: + return DerivedProperty(self._incr_func, self._incr_func(self.value)) + + def __init__(self, incr_func: Callable[[int], int], value: int) -> None: + BaseProperty.__init__(self, value) + self._incr_func = incr_func + + +class AgainProperty(DerivedProperty): + @property + def next(self) -> AgainProperty: + return AgainProperty(self._incr_func, self._incr_func(self._incr_func(self.value))) + + @property + def bad_value(self) -> int: + return self._incrementer +[out] +def BaseProperty.value(self): + self :: __main__.BaseProperty + r0 :: int + r1 :: object +L0: + r0 = self._incrementer + r1 = box(int, r0) + return r1 +def BaseProperty.bad_value(self): + self :: __main__.BaseProperty + r0 :: int + r1 :: object +L0: + r0 = self._incrementer + r1 = box(int, r0) + return r1 +def BaseProperty.next(self): + self :: __main__.BaseProperty + r0, r1 :: int + r2 :: __main__.BaseProperty +L0: + r0 = borrow self._incrementer + r1 = CPyTagged_Add(r0, 2) + keep_alive self + r2 = BaseProperty(r1) + return r2 +def BaseProperty.__init__(self, value): + self :: __main__.BaseProperty + value :: int +L0: + self._incrementer = value + return 1 +def DerivedProperty.value(self): + self :: __main__.DerivedProperty + r0 :: int +L0: + r0 = self._incrementer + return r0 +def DerivedProperty.value__BaseProperty_glue(__mypyc_self__): + __mypyc_self__ :: __main__.DerivedProperty + r0 :: int + r1 :: object +L0: + r0 = __mypyc_self__.value + r1 = box(int, r0) + return r1 +def DerivedProperty.bad_value(self): + self :: __main__.DerivedProperty + r0 :: int + r1 :: object +L0: + r0 = self._incrementer + r1 = box(int, r0) + return r1 +def DerivedProperty.next(self): + self :: __main__.DerivedProperty + r0 :: object + r1 :: int + r2, r3, r4 :: object + r5 :: int + r6 :: __main__.DerivedProperty +L0: + r0 = self._incr_func + r1 = self.value + r2 = self._incr_func + r3 = box(int, r1) + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + r5 = unbox(int, r4) + r6 = DerivedProperty(r0, r5) + return r6 +def DerivedProperty.next__BaseProperty_glue(__mypyc_self__): + __mypyc_self__, r0 :: __main__.DerivedProperty +L0: + r0 = __mypyc_self__.next + return r0 +def DerivedProperty.__init__(self, incr_func, value): + self :: __main__.DerivedProperty + incr_func :: object + value :: int + r0 :: None +L0: + r0 = BaseProperty.__init__(self, value) + self._incr_func = incr_func + return 1 +def AgainProperty.next(self): + self :: __main__.AgainProperty + r0 :: object + r1 :: int + r2, r3, r4 :: object + r5 :: int + r6, r7, r8 :: object + r9 :: int + r10 :: __main__.AgainProperty +L0: + r0 = self._incr_func + r1 = self.value + r2 = self._incr_func + r3 = box(int, r1) + r4 = PyObject_CallFunctionObjArgs(r2, r3, 0) + r5 = unbox(int, r4) + r6 = self._incr_func + r7 = box(int, r5) + r8 = PyObject_CallFunctionObjArgs(r6, r7, 0) + r9 = unbox(int, r8) + r10 = AgainProperty(r0, r9) + return r10 +def AgainProperty.next__DerivedProperty_glue(__mypyc_self__): + __mypyc_self__, r0 :: __main__.AgainProperty +L0: + r0 = __mypyc_self__.next + return r0 +def AgainProperty.next__BaseProperty_glue(__mypyc_self__): + __mypyc_self__, r0 :: __main__.AgainProperty +L0: + r0 = __mypyc_self__.next + return r0 +def AgainProperty.bad_value(self): + self :: __main__.AgainProperty + r0 :: int +L0: + r0 = self._incrementer + return r0 +def AgainProperty.bad_value__DerivedProperty_glue(__mypyc_self__): + __mypyc_self__ :: __main__.AgainProperty + r0 :: int + r1 :: object +L0: + r0 = __mypyc_self__.bad_value + r1 = box(int, r0) + return r1 +def AgainProperty.bad_value__BaseProperty_glue(__mypyc_self__): + __mypyc_self__ :: __main__.AgainProperty + r0 :: int + r1 :: object +L0: + r0 = __mypyc_self__.bad_value + r1 = box(int, r0) + return r1 + +[case testPropertyTraitSubclassing] +from mypy_extensions import trait +@trait +class SubclassedTrait: + @property + def this(self) -> SubclassedTrait: + return self + + @property + def boxed(self) -> object: + return 3 + +class DerivingObject(SubclassedTrait): + @property + def this(self) -> DerivingObject: + return self + + @property + def boxed(self) -> int: + return 5 +[out] +def SubclassedTrait.this(self): + self :: __main__.SubclassedTrait +L0: + return self +def SubclassedTrait.boxed(self): + self :: __main__.SubclassedTrait + r0 :: object +L0: + r0 = object 3 + return r0 +def DerivingObject.this(self): + self :: __main__.DerivingObject +L0: + return self +def DerivingObject.this__SubclassedTrait_glue(__mypyc_self__): + __mypyc_self__, r0 :: __main__.DerivingObject +L0: + r0 = __mypyc_self__.this + return r0 +def DerivingObject.boxed(self): + self :: __main__.DerivingObject +L0: + return 10 +def DerivingObject.boxed__SubclassedTrait_glue(__mypyc_self__): + __mypyc_self__ :: __main__.DerivingObject + r0 :: int + r1 :: object +L0: + r0 = __mypyc_self__.boxed + r1 = box(int, r0) + return r1 + +[case testI64GlueWithExtraDefaultArg] +from mypy_extensions import i64 + +class C: + def f(self) -> None: pass + +class D(C): + def f(self, x: i64 = 44) -> None: pass +[out] +def C.f(self): + self :: __main__.C +L0: + return 1 +def D.f(self, x, __bitmap): + self :: __main__.D + x :: int64 + __bitmap, r0 :: uint32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 44 +L2: + return 1 +def D.f__C_glue(self): + self :: __main__.D + r0 :: None +L0: + r0 = D.f(self, 0, 0) + return r0 + +[case testI64GlueWithSecondDefaultArg] +from mypy_extensions import i64 + +class C: + def f(self, x: i64 = 11) -> None: pass +class D(C): + def f(self, x: i64 = 12, y: i64 = 13) -> None: pass +[out] +def C.f(self, x, __bitmap): + self :: __main__.C + x :: int64 + __bitmap, r0 :: uint32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 11 +L2: + return 1 +def D.f(self, x, y, __bitmap): + self :: __main__.D + x, y :: int64 + __bitmap, r0 :: uint32 + r1 :: bit + r2 :: uint32 + r3 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 12 +L2: + r2 = __bitmap & 2 + r3 = r2 == 0 + if r3 goto L3 else goto L4 :: bool +L3: + y = 13 +L4: + return 1 +def D.f__C_glue(self, x, __bitmap): + self :: __main__.D + x :: int64 + __bitmap :: uint32 + r0 :: None +L0: + r0 = D.f(self, x, 0, __bitmap) + return r0 + +[case testI64GlueWithInvalidOverride] +from mypy_extensions import i64 + +class C: + def f(self, x: i64, y: i64 = 5) -> None: pass + def ff(self, x: int) -> None: pass +class CC(C): + def f(self, x: i64 = 12, y: i64 = 5) -> None: pass # Line 7 + def ff(self, x: int = 12) -> None: pass + +class D: + def f(self, x: int) -> None: pass +class DD(D): + def f(self, x: i64) -> None: pass # Line 13 + +class E: + def f(self, x: i64) -> None: pass +class EE(E): + def f(self, x: int) -> None: pass # Line 18 +[out] +main:7: error: An argument with type "int64" cannot be given a default value in a method override +main:13: error: Incompatible argument type "int64" (base class has type "int") +main:18: error: Incompatible argument type "int" (base class has type "int64") diff --git a/mypyc/test-data/irbuild-i32.test b/mypyc/test-data/irbuild-i32.test new file mode 100644 index 000000000000..818c3138e4e3 --- /dev/null +++ b/mypyc/test-data/irbuild-i32.test @@ -0,0 +1,482 @@ +# Test cases for i32 native ints. Focus on things that are different from i64; no need to +# duplicate all i64 test cases here. + +[case testI32BinaryOp] +from mypy_extensions import i32 + +def add_op(x: i32, y: i32) -> i32: + x = y + x + y = x + 5 + y += x + y += 7 + x = 5 + y + return x +def compare(x: i32, y: i32) -> None: + a = x == y + b = x == -5 + c = x < y + d = x < -5 + e = -5 == x + f = -5 < x +[out] +def add_op(x, y): + x, y, r0, r1, r2, r3, r4 :: int32 +L0: + r0 = y + x + x = r0 + r1 = x + 5 + y = r1 + r2 = y + x + y = r2 + r3 = y + 7 + y = r3 + r4 = 5 + y + x = r4 + return x +def compare(x, y): + x, y :: int32 + r0 :: bit + a :: bool + r1 :: bit + b :: bool + r2 :: bit + c :: bool + r3 :: bit + d :: bool + r4 :: bit + e :: bool + r5 :: bit + f :: bool +L0: + r0 = x == y + a = r0 + r1 = x == -5 + b = r1 + r2 = x < y :: signed + c = r2 + r3 = x < -5 :: signed + d = r3 + r4 = -5 == x + e = r4 + r5 = -5 < x :: signed + f = r5 + return 1 + +[case testI32UnaryOp] +from mypy_extensions import i32 + +def unary(x: i32) -> i32: + y = -x + x = ~y + y = +x + return y +[out] +def unary(x): + x, r0, y, r1 :: int32 +L0: + r0 = 0 - x + y = r0 + r1 = y ^ -1 + x = r1 + y = x + return y + +[case testI32DivisionByConstant] +from mypy_extensions import i32 + +def div_by_constant(x: i32) -> i32: + x = x // 5 + x //= 17 + return x +[out] +def div_by_constant(x): + x, r0, r1 :: int32 + r2, r3, r4 :: bit + r5 :: int32 + r6 :: bit + r7, r8, r9 :: int32 + r10, r11, r12 :: bit + r13 :: int32 + r14 :: bit + r15 :: int32 +L0: + r0 = x / 5 + r1 = r0 + r2 = x < 0 :: signed + r3 = 5 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 * 5 + r6 = r5 == x + if r6 goto L3 else goto L2 :: bool +L2: + r7 = r1 - 1 + r1 = r7 +L3: + x = r1 + r8 = x / 17 + r9 = r8 + r10 = x < 0 :: signed + r11 = 17 < 0 :: signed + r12 = r10 == r11 + if r12 goto L6 else goto L4 :: bool +L4: + r13 = r9 * 17 + r14 = r13 == x + if r14 goto L6 else goto L5 :: bool +L5: + r15 = r9 - 1 + r9 = r15 +L6: + x = r9 + return x + +[case testI32ModByConstant] +from mypy_extensions import i32 + +def mod_by_constant(x: i32) -> i32: + x = x % 5 + x %= 17 + return x +[out] +def mod_by_constant(x): + x, r0, r1 :: int32 + r2, r3, r4, r5 :: bit + r6, r7, r8 :: int32 + r9, r10, r11, r12 :: bit + r13 :: int32 +L0: + r0 = x % 5 + r1 = r0 + r2 = x < 0 :: signed + r3 = 5 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 == 0 + if r5 goto L3 else goto L2 :: bool +L2: + r6 = r1 + 5 + r1 = r6 +L3: + x = r1 + r7 = x % 17 + r8 = r7 + r9 = x < 0 :: signed + r10 = 17 < 0 :: signed + r11 = r9 == r10 + if r11 goto L6 else goto L4 :: bool +L4: + r12 = r8 == 0 + if r12 goto L6 else goto L5 :: bool +L5: + r13 = r8 + 17 + r8 = r13 +L6: + x = r8 + return x + +[case testI32DivModByVariable] +from mypy_extensions import i32 + +def divmod(x: i32, y: i32) -> i32: + a = x // y + return a % y +[out] +def divmod(x, y): + x, y, r0, a, r1 :: int32 +L0: + r0 = CPyInt32_Divide(x, y) + a = r0 + r1 = CPyInt32_Remainder(a, y) + return r1 + +[case testI32BoxAndUnbox] +from typing import Any +from mypy_extensions import i32 + +def f(x: Any) -> Any: + y: i32 = x + return y +[out] +def f(x): + x :: object + r0, y :: int32 + r1 :: object +L0: + r0 = unbox(int32, x) + y = r0 + r1 = box(int32, y) + return r1 + +[case testI32MixedCompare1_64bit] +from mypy_extensions import i32 +def f(x: int, y: i32) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: int32 + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: int32 + r7 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 4294967296 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -4294967296 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to int32 + r6 = r5 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + r7 = r6 == y + return r7 + +[case testI32MixedCompare2_64bit] +from mypy_extensions import i32 +def f(x: i32, y: int) -> bool: + return x == y +[out] +def f(x, y): + x :: int32 + y :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: int32 + r7 :: bit +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = y < 4294967296 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = y >= -4294967296 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = y >> 1 + r5 = truncate r4: native_int to int32 + r6 = r5 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + r7 = x == r6 + return r7 + +[case testI32MixedCompare_32bit] +from mypy_extensions import i32 +def f(x: int, y: i32) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: int32 + r0 :: native_int + r1 :: bit + r2, r3 :: int32 + r4 :: ptr + r5 :: c_ptr + r6 :: int32 + r7 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt32(r5) + r3 = r6 + keep_alive x +L3: + r7 = r3 == y + return r7 + +[case testI32ConvertToInt_64bit] +from mypy_extensions import i32 + +def i32_to_int(a: i32) -> int: + return a +[out] +def i32_to_int(a): + a :: int32 + r0 :: native_int + r1 :: int +L0: + r0 = extend signed a: int32 to native_int + r1 = r0 << 1 + return r1 + +[case testI32ConvertToInt_32bit] +from mypy_extensions import i32 + +def i32_to_int(a: i32) -> int: + return a +[out] +def i32_to_int(a): + a :: int32 + r0, r1 :: bit + r2, r3, r4 :: int +L0: + r0 = a <= 1073741823 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = a >= -1073741824 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromSsize_t(a) + r3 = r2 + goto L4 +L3: + r4 = a << 1 + r3 = r4 +L4: + return r3 + +[case testI32OperatorAssignmentMixed_64bit] +from mypy_extensions import i32 + +def f(a: i32) -> None: + x = 0 + x += a +[out] +def f(a): + a :: int32 + x :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6, r7 :: int32 + r8 :: native_int + r9 :: int +L0: + x = 0 + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 4294967296 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -4294967296 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to int32 + r6 = r5 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + r7 = r6 + a + r8 = extend signed r7: int32 to native_int + r9 = r8 << 1 + x = r9 + return 1 + +[case testI32InitializeFromLiteral] +from mypy_extensions import i32, i64 + +def f() -> None: + x: i32 = 0 + y: i32 = -127 + z: i32 = 5 + 7 +[out] +def f(): + x, y, z :: int32 +L0: + x = 0 + y = -127 + z = 12 + return 1 + +[case testI32ExplicitConversionFromNativeInt] +from mypy_extensions import i64, i32 + +def from_i32(x: i32) -> i32: + return i32(x) + +def from_i64(x: i64) -> i32: + return i32(x) +[out] +def from_i32(x): + x :: int32 +L0: + return x +def from_i64(x): + x :: int64 + r0 :: int32 +L0: + r0 = truncate x: int64 to int32 + return r0 + +[case testI32ExplicitConversionFromInt_64bit] +from mypy_extensions import i32 + +def f(x: int) -> i32: + return i32(x) +[out] +def f(x): + x :: int + r0 :: native_int + r1, r2, r3 :: bit + r4 :: native_int + r5, r6 :: int32 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L4 :: bool +L1: + r2 = x < 4294967296 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = x >= -4294967296 :: signed + if r3 goto L3 else goto L4 :: bool +L3: + r4 = x >> 1 + r5 = truncate r4: native_int to int32 + r6 = r5 + goto L5 +L4: + CPyInt32_Overflow() + unreachable +L5: + return r6 + +[case testI32ExplicitConversionFromLiteral] +from mypy_extensions import i32 + +def f() -> None: + x = i32(0) + y = i32(11) + z = i32(-3) +[out] +def f(): + x, y, z :: int32 +L0: + x = 0 + y = 11 + z = -3 + return 1 diff --git a/mypyc/test-data/irbuild-i64.test b/mypyc/test-data/irbuild-i64.test new file mode 100644 index 000000000000..ecedab2cd45d --- /dev/null +++ b/mypyc/test-data/irbuild-i64.test @@ -0,0 +1,1665 @@ +[case testI64Basics] +from mypy_extensions import i64 + +def f() -> i64: + x: i64 = 5 + y = x + return y +[out] +def f(): + x, y :: int64 +L0: + x = 5 + y = x + return y + +[case testI64Compare] +from mypy_extensions import i64 + +def min(x: i64, y: i64) -> i64: + if x < y: + return x + else: + return y + +def all_comparisons(x: i64) -> int: + if x == 2: + y = 10 + elif 3 != x: + y = 11 + elif x > 4: + y = 12 + elif 6 >= x: + y = 13 + elif x < 5: + y = 14 + elif 6 <= x: + y = 15 + else: + y = 16 + return y +[out] +def min(x, y): + x, y :: int64 + r0 :: bit +L0: + r0 = x < y :: signed + if r0 goto L1 else goto L2 :: bool +L1: + return x +L2: + return y +L3: + unreachable +def all_comparisons(x): + x :: int64 + r0 :: bit + y :: int + r1, r2, r3, r4, r5 :: bit +L0: + r0 = x == 2 + if r0 goto L1 else goto L2 :: bool +L1: + y = 20 + goto L18 +L2: + r1 = 3 != x + if r1 goto L3 else goto L4 :: bool +L3: + y = 22 + goto L17 +L4: + r2 = x > 4 :: signed + if r2 goto L5 else goto L6 :: bool +L5: + y = 24 + goto L16 +L6: + r3 = 6 >= x :: signed + if r3 goto L7 else goto L8 :: bool +L7: + y = 26 + goto L15 +L8: + r4 = x < 5 :: signed + if r4 goto L9 else goto L10 :: bool +L9: + y = 28 + goto L14 +L10: + r5 = 6 <= x :: signed + if r5 goto L11 else goto L12 :: bool +L11: + y = 30 + goto L13 +L12: + y = 32 +L13: +L14: +L15: +L16: +L17: +L18: + return y + +[case testI64Arithmetic] +from mypy_extensions import i64 + +def f(x: i64, y: i64) -> i64: + z = x + y + return y - z +[out] +def f(x, y): + x, y, r0, z, r1 :: int64 +L0: + r0 = x + y + z = r0 + r1 = y - z + return r1 + +[case testI64Negation] +from mypy_extensions import i64 + +def f() -> i64: + i: i64 = -3 + return -i +[out] +def f(): + i, r0 :: int64 +L0: + i = -3 + r0 = 0 - i + return r0 + +[case testI64MoreUnaryOps] +from mypy_extensions import i64 + +def unary(x: i64) -> i64: + y = ~x + x = +y + return x +[out] +def unary(x): + x, r0, y :: int64 +L0: + r0 = x ^ -1 + y = r0 + x = y + return x + +[case testI64BoxingAndUnboxing] +from typing import Any +from mypy_extensions import i64 + +def f(a: Any) -> None: + b: i64 = a + a = b +[out] +def f(a): + a :: object + r0, b :: int64 + r1 :: object +L0: + r0 = unbox(int64, a) + b = r0 + r1 = box(int64, b) + a = r1 + return 1 + +[case testI64ListGetSetItem] +from typing import List +from mypy_extensions import i64 + +def get(a: List[i64], i: i64) -> i64: + return a[i] +def set(a: List[i64], i: i64, x: i64) -> None: + a[i] = x +[out] +def get(a, i): + a :: list + i :: int64 + r0 :: object + r1 :: int64 +L0: + r0 = CPyList_GetItemInt64(a, i) + r1 = unbox(int64, r0) + return r1 +def set(a, i, x): + a :: list + i, x :: int64 + r0 :: object + r1 :: bit +L0: + r0 = box(int64, x) + r1 = CPyList_SetItemInt64(a, i, r0) + return 1 + +[case testI64MixedArithmetic] +from mypy_extensions import i64 + +def f() -> i64: + a: i64 = 1 + b = a + 2 + return 3 - b +[out] +def f(): + a, r0, b, r1 :: int64 +L0: + a = 1 + r0 = a + 2 + b = r0 + r1 = 3 - b + return r1 + +[case testI64MixedComparison] +from mypy_extensions import i64 + +def f(a: i64) -> i64: + if a < 3: + return 1 + elif 3 < a: + return 2 + return 3 +[out] +def f(a): + a :: int64 + r0, r1 :: bit +L0: + r0 = a < 3 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + r1 = 3 < a :: signed + if r1 goto L3 else goto L4 :: bool +L3: + return 2 +L4: +L5: + return 3 + +[case testI64InplaceOperations] +from mypy_extensions import i64 + +def add(a: i64) -> i64: + b = a + b += 1 + a += b + return a +def others(a: i64, b: i64) -> i64: + a -= b + a *= b + a &= b + a |= b + a ^= b + a <<= b + a >>= b + return a +[out] +def add(a): + a, b, r0, r1 :: int64 +L0: + b = a + r0 = b + 1 + b = r0 + r1 = a + b + a = r1 + return a +def others(a, b): + a, b, r0, r1, r2, r3, r4, r5, r6 :: int64 +L0: + r0 = a - b + a = r0 + r1 = a * b + a = r1 + r2 = a & b + a = r2 + r3 = a | b + a = r3 + r4 = a ^ b + a = r4 + r5 = a << b + a = r5 + r6 = a >> b + a = r6 + return a + +[case testI64BitwiseOps] +from mypy_extensions import i64 + +def forward(a: i64, b: i64) -> i64: + b = a & 1 + a = b | 2 + b = a ^ 3 + a = b << 4 + b = a >> 5 + return b + +def reverse(a: i64, b: i64) -> i64: + b = 1 & a + a = 2 | b + b = 3 ^ a + a = 4 << b + b = 5 >> a + return b + +def unary(a: i64) -> i64: + return ~a +[out] +def forward(a, b): + a, b, r0, r1, r2, r3, r4 :: int64 +L0: + r0 = a & 1 + b = r0 + r1 = b | 2 + a = r1 + r2 = a ^ 3 + b = r2 + r3 = b << 4 + a = r3 + r4 = a >> 5 + b = r4 + return b +def reverse(a, b): + a, b, r0, r1, r2, r3, r4 :: int64 +L0: + r0 = 1 & a + b = r0 + r1 = 2 | b + a = r1 + r2 = 3 ^ a + b = r2 + r3 = 4 << b + a = r3 + r4 = 5 >> a + b = r4 + return b +def unary(a): + a, r0 :: int64 +L0: + r0 = a ^ -1 + return r0 + +[case testI64Division] +from mypy_extensions import i64 + +def constant_divisor(x: i64) -> i64: + return x // 7 +def variable_divisor(x: i64, y: i64) -> i64: + return x // y +def constant_lhs(x: i64) -> i64: + return 27 // x +def divide_by_neg_one(x: i64) -> i64: + return x // -1 +def divide_by_zero(x: i64) -> i64: + return x // 0 +[out] +def constant_divisor(x): + x, r0, r1 :: int64 + r2, r3, r4 :: bit + r5 :: int64 + r6 :: bit + r7 :: int64 +L0: + r0 = x / 7 + r1 = r0 + r2 = x < 0 :: signed + r3 = 7 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 * 7 + r6 = r5 == x + if r6 goto L3 else goto L2 :: bool +L2: + r7 = r1 - 1 + r1 = r7 +L3: + return r1 +def variable_divisor(x, y): + x, y, r0 :: int64 +L0: + r0 = CPyInt64_Divide(x, y) + return r0 +def constant_lhs(x): + x, r0 :: int64 +L0: + r0 = CPyInt64_Divide(27, x) + return r0 +def divide_by_neg_one(x): + x, r0 :: int64 +L0: + r0 = CPyInt64_Divide(x, -1) + return r0 +def divide_by_zero(x): + x, r0 :: int64 +L0: + r0 = CPyInt64_Divide(x, 0) + return r0 + +[case testI64Mod] +from mypy_extensions import i64 + +def constant_divisor(x: i64) -> i64: + return x % 7 +def variable_divisor(x: i64, y: i64) -> i64: + return x % y +def constant_lhs(x: i64) -> i64: + return 27 % x +def mod_by_zero(x: i64) -> i64: + return x % 0 +[out] +def constant_divisor(x): + x, r0, r1 :: int64 + r2, r3, r4, r5 :: bit + r6 :: int64 +L0: + r0 = x % 7 + r1 = r0 + r2 = x < 0 :: signed + r3 = 7 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 == 0 + if r5 goto L3 else goto L2 :: bool +L2: + r6 = r1 + 7 + r1 = r6 +L3: + return r1 +def variable_divisor(x, y): + x, y, r0 :: int64 +L0: + r0 = CPyInt64_Remainder(x, y) + return r0 +def constant_lhs(x): + x, r0 :: int64 +L0: + r0 = CPyInt64_Remainder(27, x) + return r0 +def mod_by_zero(x): + x, r0 :: int64 +L0: + r0 = CPyInt64_Remainder(x, 0) + return r0 + +[case testI64InPlaceDiv] +from mypy_extensions import i64 + +def by_constant(x: i64) -> i64: + x //= 7 + return x +def by_variable(x: i64, y: i64) -> i64: + x //= y + return x +[out] +def by_constant(x): + x, r0, r1 :: int64 + r2, r3, r4 :: bit + r5 :: int64 + r6 :: bit + r7 :: int64 +L0: + r0 = x / 7 + r1 = r0 + r2 = x < 0 :: signed + r3 = 7 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 * 7 + r6 = r5 == x + if r6 goto L3 else goto L2 :: bool +L2: + r7 = r1 - 1 + r1 = r7 +L3: + x = r1 + return x +def by_variable(x, y): + x, y, r0 :: int64 +L0: + r0 = CPyInt64_Divide(x, y) + x = r0 + return x + +[case testI64InPlaceMod] +from mypy_extensions import i64 + +def by_constant(x: i64) -> i64: + x %= 7 + return x +def by_variable(x: i64, y: i64) -> i64: + x %= y + return x +[out] +def by_constant(x): + x, r0, r1 :: int64 + r2, r3, r4, r5 :: bit + r6 :: int64 +L0: + r0 = x % 7 + r1 = r0 + r2 = x < 0 :: signed + r3 = 7 < 0 :: signed + r4 = r2 == r3 + if r4 goto L3 else goto L1 :: bool +L1: + r5 = r1 == 0 + if r5 goto L3 else goto L2 :: bool +L2: + r6 = r1 + 7 + r1 = r6 +L3: + x = r1 + return x +def by_variable(x, y): + x, y, r0 :: int64 +L0: + r0 = CPyInt64_Remainder(x, y) + x = r0 + return x + +[case testI64ForRange] +from mypy_extensions import i64 + +def g(a: i64) -> None: pass + +def f(x: i64) -> None: + n: i64 # TODO: Infer the type + for n in range(x): + g(n) +[out] +def g(a): + a :: int64 +L0: + return 1 +def f(x): + x, r0, n :: int64 + r1 :: bit + r2 :: None + r3 :: int64 +L0: + r0 = 0 + n = r0 +L1: + r1 = r0 < x :: signed + if r1 goto L2 else goto L4 :: bool +L2: + r2 = g(n) +L3: + r3 = r0 + 1 + r0 = r3 + n = r3 + goto L1 +L4: + return 1 + +[case testI64ConvertFromInt_64bit] +from mypy_extensions import i64 + +def int_to_i64(a: int) -> i64: + return a +[out] +def int_to_i64(a): + a :: int + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6 :: int64 +L0: + r0 = a & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = a >> 1 + r3 = r2 + goto L3 +L2: + r4 = a ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive a +L3: + return r3 + +[case testI64ConvertToInt_64bit] +from mypy_extensions import i64 + +def i64_to_int(a: i64) -> int: + return a +[out] +def i64_to_int(a): + a :: int64 + r0, r1 :: bit + r2, r3, r4 :: int +L0: + r0 = a <= 4611686018427387903 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = a >= -4611686018427387904 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(a) + r3 = r2 + goto L4 +L3: + r4 = a << 1 + r3 = r4 +L4: + return r3 + +[case testI64ConvertToInt_32bit] +from mypy_extensions import i64 + +def i64_to_int(a: i64) -> int: + return a +[out] +def i64_to_int(a): + a :: int64 + r0, r1 :: bit + r2, r3 :: int + r4 :: native_int + r5 :: int +L0: + r0 = a <= 1073741823 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = a >= -1073741824 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(a) + r3 = r2 + goto L4 +L3: + r4 = truncate a: int64 to native_int + r5 = r4 << 1 + r3 = r5 +L4: + return r3 + +[case testI64Tuple] +from typing import Tuple +from mypy_extensions import i64 + +def f(x: i64, y: i64) -> Tuple[i64, i64]: + return x, y + +def g() -> Tuple[i64, i64]: + # TODO: Avoid boxing and unboxing + return 1, 2 + +def h() -> i64: + x, y = g() + t = g() + return x + y + t[0] +[out] +def f(x, y): + x, y :: int64 + r0 :: tuple[int64, int64] +L0: + r0 = (x, y) + return r0 +def g(): + r0 :: tuple[int, int] + r1 :: object + r2 :: tuple[int64, int64] +L0: + r0 = (2, 4) + r1 = box(tuple[int, int], r0) + r2 = unbox(tuple[int64, int64], r1) + return r2 +def h(): + r0 :: tuple[int64, int64] + r1, x, r2, y :: int64 + r3, t :: tuple[int64, int64] + r4, r5, r6 :: int64 +L0: + r0 = g() + r1 = r0[0] + x = r1 + r2 = r0[1] + y = r2 + r3 = g() + t = r3 + r4 = x + y + r5 = t[0] + r6 = r4 + r5 + return r6 + +[case testI64MixWithTagged1_64bit] +from mypy_extensions import i64 +def f(x: i64, y: int) -> i64: + return x + y +[out] +def f(x, y): + x :: int64 + y :: int + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6, r7 :: int64 +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = y >> 1 + r3 = r2 + goto L3 +L2: + r4 = y ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive y +L3: + r7 = x + r3 + return r7 + +[case testI64MixWithTagged2_64bit] +from mypy_extensions import i64 +def f(x: int, y: i64) -> i64: + return x + y +[out] +def f(x, y): + x :: int + y :: int64 + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6, r7 :: int64 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + r7 = r3 + y + return r7 + +[case testI64MixWithTaggedInPlace1_64bit] +from mypy_extensions import i64 +def f(y: i64) -> int: + x = 0 + x += y + return x +[out] +def f(y): + y :: int64 + x :: int + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6, r7 :: int64 + r8, r9 :: bit + r10, r11, r12 :: int +L0: + x = 0 + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + r7 = r3 + y + r8 = r7 <= 4611686018427387903 :: signed + if r8 goto L4 else goto L5 :: bool +L4: + r9 = r7 >= -4611686018427387904 :: signed + if r9 goto L6 else goto L5 :: bool +L5: + r10 = CPyTagged_FromInt64(r7) + r11 = r10 + goto L7 +L6: + r12 = r7 << 1 + r11 = r12 +L7: + x = r11 + return x + +[case testI64MixWithTaggedInPlace2_64bit] +from mypy_extensions import i64 +def f(y: int) -> i64: + x: i64 = 0 + x += y + return x +[out] +def f(y): + y :: int + x :: int64 + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6, r7 :: int64 +L0: + x = 0 + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = y >> 1 + r3 = r2 + goto L3 +L2: + r4 = y ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive y +L3: + r7 = x + r3 + x = r7 + return x + +[case testI64MixedCompare1_64bit] +from mypy_extensions import i64 +def f(x: int, y: i64) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: int64 + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6 :: int64 + r7 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + r7 = r3 == y + return r7 + +[case testI64MixedCompare2_64bit] +from mypy_extensions import i64 +def f(x: i64, y: int) -> bool: + return x == y +[out] +def f(x, y): + x :: int64 + y :: int + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6 :: int64 + r7 :: bit +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = y >> 1 + r3 = r2 + goto L3 +L2: + r4 = y ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive y +L3: + r7 = x == r3 + return r7 + +[case testI64MixedCompare_32bit] +from mypy_extensions import i64 +def f(x: int, y: i64) -> bool: + return x == y +[out] +def f(x, y): + x :: int + y :: int64 + r0 :: native_int + r1 :: bit + r2, r3, r4 :: int64 + r5 :: ptr + r6 :: c_ptr + r7 :: int64 + r8 :: bit +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = extend signed x: builtins.int to int64 + r3 = r2 >> 1 + r4 = r3 + goto L3 +L2: + r5 = x ^ 1 + r6 = r5 + r7 = CPyLong_AsInt64(r6) + r4 = r7 + keep_alive x +L3: + r8 = r4 == y + return r8 + +[case testI64AsBool] +from mypy_extensions import i64 +def f(x: i64) -> i64: + if x: + return 5 + elif not x: + return 6 + return 3 +[out] +def f(x): + x :: int64 + r0, r1 :: bit +L0: + r0 = x != 0 + if r0 goto L1 else goto L2 :: bool +L1: + return 5 +L2: + r1 = x != 0 + if r1 goto L4 else goto L3 :: bool +L3: + return 6 +L4: +L5: + return 3 + +[case testI64AssignMixed_64bit] +from mypy_extensions import i64 +def f(x: i64, y: int) -> i64: + x = y + return x +def g(x: i64, y: int) -> int: + y = x + return y +[out] +def f(x, y): + x :: int64 + y :: int + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6 :: int64 +L0: + r0 = y & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = y >> 1 + r3 = r2 + goto L3 +L2: + r4 = y ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive y +L3: + x = r3 + return x +def g(x, y): + x :: int64 + y :: int + r0, r1 :: bit + r2, r3, r4 :: int +L0: + r0 = x <= 4611686018427387903 :: signed + if r0 goto L1 else goto L2 :: bool +L1: + r1 = x >= -4611686018427387904 :: signed + if r1 goto L3 else goto L2 :: bool +L2: + r2 = CPyTagged_FromInt64(x) + r3 = r2 + goto L4 +L3: + r4 = x << 1 + r3 = r4 +L4: + y = r3 + return y + +[case testBorrowOverI64Arithmetic] +from mypy_extensions import i64 + +def add_simple(c: C) -> i64: + return c.x + c.y + +def inplace_add_simple(c: C) -> None: + c.x += c.y + +def add_borrow(d: D) -> i64: + return d.c.x + d.c.y + +class D: + c: C + +class C: + x: i64 + y: i64 +[out] +def add_simple(c): + c :: __main__.C + r0, r1, r2 :: int64 +L0: + r0 = c.x + r1 = c.y + r2 = r0 + r1 + return r2 +def inplace_add_simple(c): + c :: __main__.C + r0, r1, r2 :: int64 + r3 :: bool +L0: + r0 = c.x + r1 = c.y + r2 = r0 + r1 + c.x = r2; r3 = is_error + return 1 +def add_borrow(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int64 + r2 :: __main__.C + r3, r4 :: int64 +L0: + r0 = borrow d.c + r1 = r0.x + r2 = borrow d.c + r3 = r2.y + r4 = r1 + r3 + keep_alive d, d + return r4 + +[case testBorrowOverI64Bitwise] +from mypy_extensions import i64 + +def bitwise_simple(c: C) -> i64: + return c.x | c.y + +def inplace_bitwide_simple(c: C) -> None: + c.x &= c.y + +def bitwise_borrow(d: D) -> i64: + return d.c.x ^ d.c.y + +class D: + c: C + +class C: + x: i64 + y: i64 +[out] +def bitwise_simple(c): + c :: __main__.C + r0, r1, r2 :: int64 +L0: + r0 = c.x + r1 = c.y + r2 = r0 | r1 + return r2 +def inplace_bitwide_simple(c): + c :: __main__.C + r0, r1, r2 :: int64 + r3 :: bool +L0: + r0 = c.x + r1 = c.y + r2 = r0 & r1 + c.x = r2; r3 = is_error + return 1 +def bitwise_borrow(d): + d :: __main__.D + r0 :: __main__.C + r1 :: int64 + r2 :: __main__.C + r3, r4 :: int64 +L0: + r0 = borrow d.c + r1 = r0.x + r2 = borrow d.c + r3 = r2.y + r4 = r1 ^ r3 + keep_alive d, d + return r4 + +[case testBorrowOverI64ListGetItem1] +from mypy_extensions import i64 + +def f(n: i64) -> str: + a = [C()] + return a[n].s + +class C: + s: str +[out] +def f(n): + n :: int64 + r0 :: __main__.C + r1 :: list + r2, r3 :: ptr + a :: list + r4 :: object + r5 :: __main__.C + r6 :: str +L0: + r0 = C() + r1 = PyList_New(1) + r2 = get_element_ptr r1 ob_item :: PyListObject + r3 = load_mem r2 :: ptr* + set_mem r3, r0 :: builtins.object* + keep_alive r1 + a = r1 + r4 = CPyList_GetItemInt64Borrow(a, n) + r5 = borrow cast(__main__.C, r4) + r6 = r5.s + keep_alive a, n, r4 + return r6 + +[case testBorrowOverI64ListGetItem2] +from typing import List +from mypy_extensions import i64 + +def f(a: List[i64], n: i64) -> bool: + if a[n] == 0: + return True + return False +[out] +def f(a, n): + a :: list + n :: int64 + r0 :: object + r1 :: int64 + r2 :: bit +L0: + r0 = CPyList_GetItemInt64Borrow(a, n) + r1 = unbox(int64, r0) + r2 = r1 == 0 + keep_alive a, n + if r2 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + +[case testCoerceShortIntToI64] +from mypy_extensions import i64 +from typing import List + +def f(a: List[i64], y: i64) -> bool: + if len(a) < y: + return True + return False + +def g(a: List[i64], y: i64) -> bool: + if y < len(a): + return True + return False +[out] +def f(a, y): + a :: list + y :: int64 + r0 :: ptr + r1 :: native_int + r2 :: short_int + r3 :: int64 + r4 :: bit +L0: + r0 = get_element_ptr a ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive a + r2 = r1 << 1 + r3 = r2 >> 1 + r4 = r3 < y :: signed + if r4 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 +def g(a, y): + a :: list + y :: int64 + r0 :: ptr + r1 :: native_int + r2 :: short_int + r3 :: int64 + r4 :: bit +L0: + r0 = get_element_ptr a ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive a + r2 = r1 << 1 + r3 = r2 >> 1 + r4 = y < r3 :: signed + if r4 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + +[case testMultiplyListByI64_64bit] +from mypy_extensions import i64 +from typing import List + +def f(n: i64) -> List[i64]: + return [n] * n +[out] +def f(n): + n :: int64 + r0 :: list + r1 :: object + r2, r3 :: ptr + r4, r5 :: bit + r6, r7, r8 :: int + r9 :: list +L0: + r0 = PyList_New(1) + r1 = box(int64, n) + r2 = get_element_ptr r0 ob_item :: PyListObject + r3 = load_mem r2 :: ptr* + set_mem r3, r1 :: builtins.object* + keep_alive r0 + r4 = n <= 4611686018427387903 :: signed + if r4 goto L1 else goto L2 :: bool +L1: + r5 = n >= -4611686018427387904 :: signed + if r5 goto L3 else goto L2 :: bool +L2: + r6 = CPyTagged_FromInt64(n) + r7 = r6 + goto L4 +L3: + r8 = n << 1 + r7 = r8 +L4: + r9 = CPySequence_Multiply(r0, r7) + return r9 + +[case testShortIntAndI64Op] +from mypy_extensions import i64 +from typing import List + +def add_i64(a: List[i64], n: i64) -> i64: + return len(a) + n +def add_i64_2(a: List[i64], n: i64) -> i64: + return n + len(a) +def eq_i64(a: List[i64], n: i64) -> bool: + if len(a) == n: + return True + return False +def lt_i64(a: List[i64], n: i64) -> bool: + if n < len(a): + return True + return False +[out] +def add_i64(a, n): + a :: list + n :: int64 + r0 :: ptr + r1 :: native_int + r2 :: short_int + r3, r4 :: int64 +L0: + r0 = get_element_ptr a ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive a + r2 = r1 << 1 + r3 = r2 >> 1 + r4 = r3 + n + return r4 +def add_i64_2(a, n): + a :: list + n :: int64 + r0 :: ptr + r1 :: native_int + r2 :: short_int + r3, r4 :: int64 +L0: + r0 = get_element_ptr a ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive a + r2 = r1 << 1 + r3 = r2 >> 1 + r4 = n + r3 + return r4 +def eq_i64(a, n): + a :: list + n :: int64 + r0 :: ptr + r1 :: native_int + r2 :: short_int + r3 :: int64 + r4 :: bit +L0: + r0 = get_element_ptr a ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive a + r2 = r1 << 1 + r3 = r2 >> 1 + r4 = r3 == n + if r4 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 +def lt_i64(a, n): + a :: list + n :: int64 + r0 :: ptr + r1 :: native_int + r2 :: short_int + r3 :: int64 + r4 :: bit +L0: + r0 = get_element_ptr a ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive a + r2 = r1 << 1 + r3 = r2 >> 1 + r4 = n < r3 :: signed + if r4 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + return 0 + +[case testOptionalI64_64bit] +from typing import Optional +from mypy_extensions import i64 + +def f(x: Optional[i64]) -> i64: + if x is None: + return 1 + return x +[out] +def f(x): + x :: union[int64, None] + r0 :: object + r1 :: bit + r2 :: int64 +L0: + r0 = load_address _Py_NoneStruct + r1 = x == r0 + if r1 goto L1 else goto L2 :: bool +L1: + return 1 +L2: + r2 = unbox(int64, x) + return r2 + +[case testI64DefaultValueSingle] +from mypy_extensions import i64 + +def f(x: i64, y: i64 = 0) -> i64: + return x + y + +def g() -> i64: + return f(7) + f(8, 9) +[out] +def f(x, y, __bitmap): + x, y :: int64 + __bitmap, r0 :: uint32 + r1 :: bit + r2 :: int64 +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + y = 0 +L2: + r2 = x + y + return r2 +def g(): + r0, r1, r2 :: int64 +L0: + r0 = f(7, 0, 0) + r1 = f(8, 9, 1) + r2 = r0 + r1 + return r2 + +[case testI64DefaultValueWithMultipleArgs] +from mypy_extensions import i64 + +def f(a: i64, b: i64 = 1, c: int = 2, d: i64 = 3) -> i64: + return 0 + +def g() -> i64: + return f(7) + f(8, 9) + f(1, 2, 3) + f(4, 5, 6, 7) +[out] +def f(a, b, c, d, __bitmap): + a, b :: int64 + c :: int + d :: int64 + __bitmap, r0 :: uint32 + r1 :: bit + r2 :: uint32 + r3 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + b = 1 +L2: + if is_error(c) goto L3 else goto L4 +L3: + c = 4 +L4: + r2 = __bitmap & 2 + r3 = r2 == 0 + if r3 goto L5 else goto L6 :: bool +L5: + d = 3 +L6: + return 0 +def g(): + r0 :: int + r1 :: int64 + r2 :: int + r3, r4, r5, r6, r7, r8 :: int64 +L0: + r0 = :: int + r1 = f(7, 0, r0, 0, 0) + r2 = :: int + r3 = f(8, 9, r2, 0, 1) + r4 = r1 + r3 + r5 = f(1, 2, 6, 0, 1) + r6 = r4 + r5 + r7 = f(4, 5, 12, 7, 3) + r8 = r6 + r7 + return r8 + +[case testI64MethodDefaultValue] +from mypy_extensions import i64 + +class C: + def m(self, x: i64 = 5) -> None: + pass + +def f(c: C) -> None: + c.m() + c.m(6) +[out] +def C.m(self, x, __bitmap): + self :: __main__.C + x :: int64 + __bitmap, r0 :: uint32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 5 +L2: + return 1 +def f(c): + c :: __main__.C + r0, r1 :: None +L0: + r0 = c.m(0, 0) + r1 = c.m(6, 1) + return 1 + +[case testI64ExplicitConversionFromNativeInt] +from mypy_extensions import i64, i32 + +def from_i32(x: i32) -> i64: + return i64(x) + +def from_i64(x: i64) -> i64: + return i64(x) +[out] +def from_i32(x): + x :: int32 + r0 :: int64 +L0: + r0 = extend signed x: int32 to int64 + return r0 +def from_i64(x): + x :: int64 +L0: + return x + +[case testI64ExplicitConversionFromInt_64bit] +from mypy_extensions import i64 + +def f(x: int) -> i64: + return i64(x) +[out] +def f(x): + x :: int + r0 :: native_int + r1 :: bit + r2, r3 :: int64 + r4 :: ptr + r5 :: c_ptr + r6 :: int64 +L0: + r0 = x & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = x >> 1 + r3 = r2 + goto L3 +L2: + r4 = x ^ 1 + r5 = r4 + r6 = CPyLong_AsInt64(r5) + r3 = r6 + keep_alive x +L3: + return r3 + +[case testI64ExplicitConversionFromLiteral] +from mypy_extensions import i64 + +def f() -> None: + x = i64(0) + y = i64(11) + z = i64(-3) +[out] +def f(): + x, y, z :: int64 +L0: + x = 0 + y = 11 + z = -3 + return 1 + +[case testI64ForLoopOverRange] +from mypy_extensions import i64 + +def f() -> None: + for x in range(i64(4)): + y = x +[out] +def f(): + r0, x :: int64 + r1 :: bit + y, r2 :: int64 +L0: + r0 = 0 + x = r0 +L1: + r1 = r0 < 4 :: signed + if r1 goto L2 else goto L4 :: bool +L2: + y = x +L3: + r2 = r0 + 1 + r0 = r2 + x = r2 + goto L1 +L4: + return 1 + +[case testI64ForLoopOverRange2] +from mypy_extensions import i64 + +def f() -> None: + for x in range(0, i64(4)): + y = x +[out] +def f(): + r0, x :: int64 + r1 :: bit + y, r2 :: int64 +L0: + r0 = 0 + x = r0 +L1: + r1 = r0 < 4 :: signed + if r1 goto L2 else goto L4 :: bool +L2: + y = x +L3: + r2 = r0 + 1 + r0 = r2 + x = r2 + goto L1 +L4: + return 1 + +[case testI64MethodDefaultValueOverride] +from mypy_extensions import i64 + +class C: + def f(self, x: i64 = 11) -> None: pass +class D(C): + def f(self, x: i64 = 12) -> None: pass +[out] +def C.f(self, x, __bitmap): + self :: __main__.C + x :: int64 + __bitmap, r0 :: uint32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 11 +L2: + return 1 +def D.f(self, x, __bitmap): + self :: __main__.D + x :: int64 + __bitmap, r0 :: uint32 + r1 :: bit +L0: + r0 = __bitmap & 1 + r1 = r0 == 0 + if r1 goto L1 else goto L2 :: bool +L1: + x = 12 +L2: + return 1 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index ce365fc50e7e..372956a00cab 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1490,3 +1490,39 @@ L0: r2 = CPyTagged_Subtract(r0, r1) c.x = r2; r3 = is_error return 1 + +[case testCoerceIntToI64_64bit] +from mypy_extensions import i64 + +def f(x: int) -> i64: + # TODO: On the fast path we shouldn't have a decref. Once we have high-level IR, + # coercion from int to i64 can be a single op, which makes it easier to + # generate optimal refcount handling for this case. + return x + 1 +[out] +def f(x): + x, r0 :: int + r1 :: native_int + r2 :: bit + r3, r4 :: int64 + r5 :: ptr + r6 :: c_ptr + r7 :: int64 +L0: + r0 = CPyTagged_Add(x, 2) + r1 = r0 & 1 + r2 = r1 == 0 + if r2 goto L1 else goto L2 :: bool +L1: + r3 = r0 >> 1 + dec_ref r0 :: int + r4 = r3 + goto L3 +L2: + r5 = r0 ^ 1 + r6 = r5 + r7 = CPyLong_AsInt64(r6) + r4 = r7 + dec_ref r0 :: int +L3: + return r4 diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 0ed7b2c7fd2d..d505bda2d705 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -1774,6 +1774,36 @@ Represents a sequence of values. Updates itself by next, which is a new value. Represents a sequence of values. Updates itself by next, which is a new value. 3 3 +[out version>=3.11] +Traceback (most recent call last): + File "driver.py", line 5, in + print (x.rankine) + ^^^^^^^^^ + File "native.py", line 16, in rankine + raise NotImplementedError +NotImplementedError +0.0 +F: 32.0 C: 0.0 +100.0 +F: 212.0 C: 100.0 +1 +2 +3 +4 + [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + [7, 22, 11, 34, 17, 52, 26, 13, 40, 20, 10, 5, 16, 8, 4, 2, 1, 4, 2, 1] + [7, 11, 17, 26, 40, 10, 16, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4] +10 +34 +26 + [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + [7, 22, 11, 34, 17, 52, 26, 13, 40, 20, 10, 5, 16, 8, 4, 2, 1, 4, 2, 1] + [7, 11, 17, 26, 40, 10, 16, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4, 1, 2, 4] +Represents a sequence of values. Updates itself by next, which is a new value. +Represents a sequence of values. Updates itself by next, which is a new value. +Represents a sequence of values. Updates itself by next, which is a new value. +3 +3 [case testPropertySetters] diff --git a/mypyc/test-data/run-dunders.test b/mypyc/test-data/run-dunders.test index aee2a956c47f..0b156e5c3af8 100644 --- a/mypyc/test-data/run-dunders.test +++ b/mypyc/test-data/run-dunders.test @@ -332,6 +332,13 @@ class C: def __float__(self) -> float: return float(self.x + 4) + def __pos__(self) -> int: + return self.x + 5 + + def __abs__(self) -> int: + return abs(self.x) + 6 + + def test_unary_dunders_generic() -> None: a: Any = C(10) @@ -339,6 +346,8 @@ def test_unary_dunders_generic() -> None: assert ~a == 12 assert int(a) == 13 assert float(a) == 14.0 + assert +a == 15 + assert abs(a) == 16 def test_unary_dunders_native() -> None: c = C(10) @@ -347,6 +356,8 @@ def test_unary_dunders_native() -> None: assert ~c == 12 assert int(c) == 13 assert float(c) == 14.0 + assert +c == 15 + assert abs(c) == 16 [case testDundersBinarySimple] from typing import Any diff --git a/mypyc/test-data/run-functions.test b/mypyc/test-data/run-functions.test index b6277c9e8ec4..21993891c4e3 100644 --- a/mypyc/test-data/run-functions.test +++ b/mypyc/test-data/run-functions.test @@ -430,9 +430,11 @@ def nested_funcs(n: int) -> List[Callable[..., Any]]: ls.append(f) return ls +def bool_default(x: bool = False, y: bool = True) -> str: + return str(x) + '-' + str(y) [file driver.py] -from native import f, g, h, same, nested_funcs, a_lambda +from native import f, g, h, same, nested_funcs, a_lambda, bool_default g() assert f(2) == (5, "test") assert f(s = "123", x = -2) == (1, "123") @@ -447,6 +449,10 @@ assert [f() for f in nested_funcs(10)] == list(range(10)) assert a_lambda(10) == 10 assert a_lambda() == 20 +assert bool_default() == 'False-True' +assert bool_default(True) == 'True-True' +assert bool_default(True, False) == 'True-False' + [case testMethodCallWithDefaultArgs] from typing import Tuple, List class A: @@ -1235,3 +1241,18 @@ def g() -> None: a.pop() g() + +[case testUnpackKwargsCompiled] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]) -> None: + print(kwargs["name"]) + +# This is not really supported yet, just test that we behave reasonably. +foo(name='Jennifer', age=38) +[out] +Jennifer diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index db658eea6504..0f2cbe152fc0 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -650,3 +650,15 @@ from testutil import run_generator yields, val = run_generator(finally_yield()) assert yields == ('x',) assert val == 'test', val + +[case testUnreachableComprehensionNoCrash] +from typing import List + +def list_comp() -> List[int]: + if True: + return [5] + return [i for i in [5]] + +[file driver.py] +from native import list_comp +assert list_comp() == [5] diff --git a/mypyc/test-data/run-i32.test b/mypyc/test-data/run-i32.test new file mode 100644 index 000000000000..3d2f3e59e83c --- /dev/null +++ b/mypyc/test-data/run-i32.test @@ -0,0 +1,318 @@ +[case testI32BasicOps] +from typing import Any, Tuple + +MYPY = False +if MYPY: + from mypy_extensions import i32, i64 + +from testutil import assertRaises + +def test_box_and_unbox() -> None: + values = (list(range(-2**31, -2**31 + 100)) + + list(range(-1000, 1000)) + + list(range(2**31 - 100, 2**31))) + for i in values: + o: Any = i + x: i32 = o + o2: Any = x + assert o == o2 + assert x == i + with assertRaises(OverflowError, "int too large to convert to i32"): + o = 2**31 + x2: i32 = o + with assertRaises(OverflowError, "int too large to convert to i32"): + o = -2**32 - 1 + x3: i32 = o + +def div_by_7(x: i32) -> i32: + return x // 7 +def div_by_neg_7(x: i32) -> i32: + return x // -7 + +def div(x: i32, y: i32) -> i32: + return x // y + +def test_divide_by_constant() -> None: + for i in range(-1000, 1000): + assert div_by_7(i) == i // 7 + for i in range(-2**31, -2**31 + 1000): + assert div_by_7(i) == i // 7 + for i in range(2**31 - 1000, 2**31): + assert div_by_7(i) == i // 7 + +def test_divide_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert div_by_neg_7(i) == i // -7 + for i in range(-2**31, -2**31 + 1000): + assert div_by_neg_7(i) == i // -7 + for i in range(2**31 - 1000, 2**31): + assert div_by_neg_7(i) == i // -7 + +def test_divide_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**31, -2**31 + 10)) + + list(range(2**31 - 10, 2**31))) + for x in values: + for y in values: + if y != 0: + if x // y == 2**31: + with assertRaises(OverflowError, "integer division overflow"): + div(x, y) + else: + assert div(x, y) == x // y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + div(x, y) + +def mod_by_7(x: i32) -> i32: + return x % 7 + +def mod_by_neg_7(x: i32) -> i32: + return x // -7 + +def mod(x: i32, y: i32) -> i32: + return x % y + +def test_mod_by_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_7(i) == i % 7 + for i in range(-2**31, -2**31 + 1000): + assert mod_by_7(i) == i % 7 + for i in range(2**31 - 1000, 2**31): + assert mod_by_7(i) == i % 7 + +def test_mod_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(-2**31, -2**31 + 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(2**31 - 1000, 2**31): + assert mod_by_neg_7(i) == i // -7 + +def test_mod_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**31, -2**31 + 10)) + + list(range(2**31 - 10, 2**31))) + for x in values: + for y in values: + if y != 0: + assert mod(x, y) == x % y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + mod(x, y) + +def test_simple_arithmetic_ops() -> None: + zero: i32 = int() + one: i32 = zero + 1 + two: i32 = one + 1 + neg_one: i32 = -one + assert one + one == 2 + assert one + two == 3 + assert one + neg_one == 0 + assert one - one == 0 + assert one - two == -1 + assert one * one == 1 + assert one * two == 2 + assert two * two == 4 + assert two * neg_one == -2 + assert neg_one * one == -1 + assert neg_one * neg_one == 1 + assert two * 0 == 0 + assert 0 * two == 0 + assert -one == -1 + assert -two == -2 + assert -neg_one == 1 + assert -zero == 0 + +def test_bitwise_ops() -> None: + x: i32 = 1920687484 + int() + y: i32 = 383354614 + int() + z: i32 = -1879040563 + int() + zero: i32 = int() + one: i32 = zero + 1 + two: i32 = zero + 2 + neg_one: i32 = -one + + assert x & y == 307823732 + assert x & z == 268442956 + assert z & z == z + assert x & zero == 0 + + assert x | y == 1996218366 + assert x | z == -226796035 + assert z | z == z + assert x | 0 == x + + assert x ^ y == 1688394634 + assert x ^ z == -495238991 + assert z ^ z == 0 + assert z ^ 0 == z + + assert x << one == -453592328 + assert x << two == -907184656 + assert z << two == 1073772340 + assert z << 0 == z + + assert x >> one == 960343742 + assert x >> two == 480171871 + assert z >> two == -469760141 + assert z >> 0 == z + + assert ~x == -1920687485 + assert ~z == 1879040562 + assert ~zero == -1 + assert ~neg_one == 0 + +def eq(x: i32, y: i32) -> bool: + return x == y + +def test_eq() -> None: + assert eq(int(), int()) + assert eq(5 + int(), 5 + int()) + assert eq(-5 + int(), -5 + int()) + assert not eq(int(), 1 + int()) + assert not eq(5 + int(), 6 + int()) + assert not eq(-5 + int(), -6 + int()) + assert not eq(-5 + int(), 5 + int()) + +def test_comparisons() -> None: + one: i32 = 1 + int() + one2: i32 = 1 + int() + two: i32 = 2 + int() + assert one < two + assert not (one < one2) + assert not (two < one) + assert two > one + assert not (one > one2) + assert not (one > two) + assert one <= two + assert one <= one2 + assert not (two <= one) + assert two >= one + assert one >= one2 + assert not (one >= two) + assert one == one2 + assert not (one == two) + assert one != two + assert not (one != one2) + +def test_mixed_comparisons() -> None: + i32_3: i32 = int() + 3 + int_5 = int() + 5 + assert i32_3 < int_5 + assert int_5 > i32_3 + b = i32_3 > int_5 + assert not b + + int_largest = int() + (1 << 31) - 1 + assert int_largest > i32_3 + int_smallest = int() - (1 << 31) + assert i32_3 > int_smallest + + int_too_big = int() + (1 << 31) + int_too_small = int() - (1 << 31) - 1 + with assertRaises(OverflowError): + assert i32_3 < int_too_big + with assertRaises(OverflowError): + assert int_too_big < i32_3 + with assertRaises(OverflowError): + assert i32_3 > int_too_small + with assertRaises(OverflowError): + assert int_too_small < i32_3 + +def test_mixed_arithmetic_and_bitwise_ops() -> None: + i32_3: i32 = int() + 3 + int_5 = int() + 5 + assert i32_3 + int_5 == 8 + assert int_5 - i32_3 == 2 + assert i32_3 << int_5 == 96 + assert int_5 << i32_3 == 40 + assert i32_3 ^ int_5 == 6 + assert int_5 | i32_3 == 7 + + int_largest = int() + (1 << 31) - 1 + assert int_largest - i32_3 == 2147483644 + int_smallest = int() - (1 << 31) + assert int_smallest + i32_3 == -2147483645 + + int_too_big = int() + (1 << 31) + int_too_small = int() - (1 << 31) - 1 + with assertRaises(OverflowError): + assert i32_3 & int_too_big + with assertRaises(OverflowError): + assert int_too_small & i32_3 + +def test_coerce_to_and_from_int() -> None: + for shift in range(0, 32): + for sign in 1, -1: + for delta in range(-5, 5): + n = sign * (1 << shift) + delta + if -(1 << 31) <= n < (1 << 31): + x: i32 = n + m: int = x + assert m == n + +def test_explicit_conversion_to_i32() -> None: + x = i32(5) + assert x == 5 + y = int() - 113 + x = i32(y) + assert x == -113 + n64: i64 = 1733 + x = i32(n64) + assert x == 1733 + n32 = -1733 + x = i32(n32) + assert x == -1733 + z = i32(x) + assert z == -1733 + +def test_explicit_conversion_overflow() -> None: + max_i32 = int() + 2**31 - 1 + x = i32(max_i32) + assert x == 2**31 - 1 + assert int(x) == max_i32 + + min_i32 = int() - 2**31 + y = i32(min_i32) + assert y == -2**31 + assert int(y) == min_i32 + + too_big = int() + 2**31 + with assertRaises(OverflowError): + x = i32(too_big) + + too_small = int() - 2**31 - 1 + with assertRaises(OverflowError): + x = i32(too_small) + +def test_i32_from_large_small_literal() -> None: + x = i32(2**31 - 1) + assert x == 2**31 - 1 + x = i32(-2**31) + assert x == -2**31 + +def test_i32_truncate_from_i64() -> None: + large = i64(2**32 + 157 + int()) + x = i32(large) + assert x == 157 + small = i64(-2**32 - 157 + int()) + x = i32(small) + assert x == -157 + large2 = i64(2**31 + int()) + x = i32(large2) + assert x == -2**31 + small2 = i64(-2**31 - 1 - int()) + x = i32(small2) + assert x == 2**31 - 1 + +def test_tuple_i32() -> None: + a: i32 = 1 + b: i32 = 2 + t = (a, b) + a, b = t + assert a == 1 + assert b == 2 + x: Any = t + tt: Tuple[i32, i32] = x + assert tt == (1, 2) diff --git a/mypyc/test-data/run-i64.test b/mypyc/test-data/run-i64.test new file mode 100644 index 000000000000..ee0a09760ab1 --- /dev/null +++ b/mypyc/test-data/run-i64.test @@ -0,0 +1,1172 @@ +[case testI64BasicOps] +from typing import List, Any, Tuple + +MYPY = False +if MYPY: + from mypy_extensions import i64, i32 + +from testutil import assertRaises + +def inc(n: i64) -> i64: + return n + 1 + +def test_inc() -> None: + # Use int() to avoid constant folding + n = 1 + int() + m = 2 + int() + assert inc(n) == m + +def min_ll(x: i64, y: i64) -> i64: + if x < y: + return x + else: + return y + +def test_min() -> None: + assert min_ll(1 + int(), 2) == 1 + assert min_ll(2 + int(), 1) == 1 + assert min_ll(1 + int(), 1) == 1 + assert min_ll(-2 + int(), 1) == -2 + assert min_ll(1 + int(), -2) == -2 + +def eq(x: i64, y: i64) -> bool: + return x == y + +def test_eq() -> None: + assert eq(int(), int()) + assert eq(5 + int(), 5 + int()) + assert eq(-5 + int(), -5 + int()) + assert not eq(int(), 1 + int()) + assert not eq(5 + int(), 6 + int()) + assert not eq(-5 + int(), -6 + int()) + assert not eq(-5 + int(), 5 + int()) + +def test_comparisons() -> None: + one: i64 = 1 + int() + one2: i64 = 1 + int() + two: i64 = 2 + int() + assert one < two + assert not (one < one2) + assert not (two < one) + assert two > one + assert not (one > one2) + assert not (one > two) + assert one <= two + assert one <= one2 + assert not (two <= one) + assert two >= one + assert one >= one2 + assert not (one >= two) + assert one == one2 + assert not (one == two) + assert one != two + assert not (one != one2) + +def div_by_3(x: i64) -> i64: + return x // 3 + +def div_by_neg_3(x: i64) -> i64: + return x // -3 + +def div(x: i64, y: i64) -> i64: + return x // y + +def test_divide_by_constant() -> None: + for i in range(-1000, 1000): + assert div_by_3(i) == i // 3 + for i in range(-2**63, -2**63 + 1000): + assert div_by_3(i) == i // 3 + for i in range(2**63 - 1000, 2**63): + assert div_by_3(i) == i // 3 + +def test_divide_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert div_by_neg_3(i) == i // -3 + for i in range(-2**63, -2**63 + 1000): + assert div_by_neg_3(i) == i // -3 + for i in range(2**63 - 1000, 2**63): + assert div_by_neg_3(i) == i // -3 + +def test_divide_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**63, -2**63 + 10)) + + list(range(2**63 - 10, 2**63))) + for x in values: + for y in values: + if y != 0: + if x // y == 2**63: + with assertRaises(OverflowError, "integer division overflow"): + div(x, y) + else: + assert div(x, y) == x // y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + div(x, y) + +def mod_by_7(x: i64) -> i64: + return x % 7 + +def mod_by_neg_7(x: i64) -> i64: + return x // -7 + +def mod(x: i64, y: i64) -> i64: + return x % y + +def test_mod_by_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_7(i) == i % 7 + for i in range(-2**63, -2**63 + 1000): + assert mod_by_7(i) == i % 7 + for i in range(2**63 - 1000, 2**63): + assert mod_by_7(i) == i % 7 + +def test_mod_by_negative_constant() -> None: + for i in range(-1000, 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(-2**63, -2**63 + 1000): + assert mod_by_neg_7(i) == i // -7 + for i in range(2**63 - 1000, 2**63): + assert mod_by_neg_7(i) == i // -7 + +def test_mod_by_variable() -> None: + values = (list(range(-50, 50)) + + list(range(-2**63, -2**63 + 10)) + + list(range(2**63 - 10, 2**63))) + for x in values: + for y in values: + if y != 0: + assert mod(x, y) == x % y + else: + with assertRaises(ZeroDivisionError, "integer division or modulo by zero"): + mod(x, y) + +def get_item(a: List[i64], n: i64) -> i64: + return a[n] + +def test_get_list_item() -> None: + a = [1, 6, -2] + assert get_item(a, 0) == 1 + assert get_item(a, 1) == 6 + assert get_item(a, 2) == -2 + assert get_item(a, -1) == -2 + assert get_item(a, -2) == 6 + assert get_item(a, -3) == 1 + with assertRaises(IndexError, "list index out of range"): + get_item(a, 3) + with assertRaises(IndexError, "list index out of range"): + get_item(a, -4) + # TODO: Very large/small values and indexes + +def test_simple_arithmetic_ops() -> None: + zero: i64 = int() + one: i64 = zero + 1 + two: i64 = one + 1 + neg_one: i64 = -one + assert one + one == 2 + assert one + two == 3 + assert one + neg_one == 0 + assert one - one == 0 + assert one - two == -1 + assert one * one == 1 + assert one * two == 2 + assert two * two == 4 + assert two * neg_one == -2 + assert neg_one * one == -1 + assert neg_one * neg_one == 1 + assert two * 0 == 0 + assert 0 * two == 0 + assert -one == -1 + assert -two == -2 + assert -neg_one == 1 + assert -zero == 0 + +def test_bitwise_ops() -> None: + x: i64 = 7997307308812232241 + int() + y: i64 = 4333433528471475340 + int() + z: i64 = -2462230749488444526 + int() + zero: i64 = int() + one: i64 = zero + 1 + two: i64 = zero + 2 + neg_one: i64 = -one + + assert x & y == 3179577071592752128 + assert x & z == 5536089561888850448 + assert z & z == z + assert x & zero == 0 + + assert x | y == 9151163765690955453 + assert x | z == -1013002565062733 + assert z | z == z + assert x | 0 == x + + assert x ^ y == 5971586694098203325 + assert x ^ z == -5537102564453913181 + assert z ^ z == 0 + assert z ^ 0 == z + + assert x << one == -2452129456085087134 + assert x << two == -4904258912170174268 + assert z << two == 8597821075755773512 + assert z << 0 == z + + assert x >> one == 3998653654406116120 + assert x >> two == 1999326827203058060 + assert z >> two == -615557687372111132 + assert z >> 0 == z + + assert ~x == -7997307308812232242 + assert ~z == 2462230749488444525 + assert ~zero == -1 + assert ~neg_one == 0 + +def test_coerce_to_and_from_int() -> None: + for shift in range(0, 64): + for sign in 1, -1: + for delta in range(-5, 5): + n = sign * (1 << shift) + delta + if -(1 << 63) <= n < (1 << 63): + x: i64 = n + m: int = x + assert m == n + +def test_explicit_conversion_to_i64() -> None: + x = i64(5) + assert x == 5 + y = int() - 113 + x = i64(y) + assert x == -113 + n32: i32 = 1733 + x = i64(n32) + assert x == 1733 + n32 = -1733 + x = i64(n32) + assert x == -1733 + z = i64(x) + assert z == -1733 + +def test_explicit_conversion_overflow() -> None: + max_i64 = int() + 2**63 - 1 + x = i64(max_i64) + assert x == 2**63 - 1 + assert int(x) == max_i64 + + min_i64 = int() - 2**63 + y = i64(min_i64) + assert y == -2**63 + assert int(y) == min_i64 + + too_big = int() + 2**63 + with assertRaises(OverflowError): + x = i64(too_big) + + too_small = int() - 2**63 - 1 + with assertRaises(OverflowError): + x = i64(too_small) + +def test_i64_from_large_small_literal() -> None: + x = i64(2**63 - 1) + assert x == 2**63 - 1 + x = i64(-2**63) + assert x == -2**63 + +def test_tuple_i64() -> None: + a: i64 = 1 + b: i64 = 2 + t = (a, b) + a, b = t + assert a == 1 + assert b == 2 + x: Any = t + tt: Tuple[i64, i64] = x + assert tt == (1, 2) + +def test_list_set_item() -> None: + a: List[i64] = [0, 2, 6] + z: i64 = int() + a[z] = 1 + assert a == [1, 2, 6] + a[z + 2] = 9 + assert a == [1, 2, 9] + a[-(z + 1)] = 10 + assert a == [1, 2, 10] + a[-(z + 3)] = 3 + assert a == [3, 2, 10] + with assertRaises(IndexError): + a[z + 3] = 0 + with assertRaises(IndexError): + a[-(z + 4)] = 0 + assert a == [3, 2, 10] + +class C: + def __init__(self, x: i64) -> None: + self.x = x + +def test_attributes() -> None: + i: i64 + for i in range(-1000, 1000): + c = C(i) + assert c.x == i + c.x = i + 1 + assert c.x == i + 1 + +def test_mixed_comparisons() -> None: + i64_3: i64 = int() + 3 + int_5 = int() + 5 + assert i64_3 < int_5 + assert int_5 > i64_3 + b = i64_3 > int_5 + assert not b + + int_largest = int() + (1 << 63) - 1 + assert int_largest > i64_3 + int_smallest = int() - (1 << 63) + assert i64_3 > int_smallest + + int_too_big = int() + (1 << 63) + int_too_small = int() - (1 << 63) - 1 + with assertRaises(OverflowError): + assert i64_3 < int_too_big + with assertRaises(OverflowError): + assert int_too_big < i64_3 + with assertRaises(OverflowError): + assert i64_3 > int_too_small + with assertRaises(OverflowError): + assert int_too_small < i64_3 + +def test_mixed_comparisons_32bit() -> None: + # Test edge cases on 32-bit platforms + i64_3: i64 = int() + 3 + int_5 = int() + 5 + + int_largest_short = int() + (1 << 30) - 1 + int_largest_short_i64: i64 = int_largest_short + assert int_largest_short > i64_3 + int_smallest_short = int() - (1 << 30) + int_smallest_short_i64: i64 = int_smallest_short + assert i64_3 > int_smallest_short + + int_big = int() + (1 << 30) + assert int_big > i64_3 + int_small = int() - (1 << 30) - 1 + assert i64_3 > int_small + + assert int_smallest_short_i64 > int_small + assert int_largest_short_i64 < int_big + +def test_mixed_arithmetic_and_bitwise_ops() -> None: + i64_3: i64 = int() + 3 + int_5 = int() + 5 + assert i64_3 + int_5 == 8 + assert int_5 - i64_3 == 2 + assert i64_3 << int_5 == 96 + assert int_5 << i64_3 == 40 + assert i64_3 ^ int_5 == 6 + assert int_5 | i64_3 == 7 + + int_largest = int() + (1 << 63) - 1 + assert int_largest - i64_3 == 9223372036854775804 + int_smallest = int() - (1 << 63) + assert int_smallest + i64_3 == -9223372036854775805 + + int_too_big = int() + (1 << 63) + int_too_small = int() - (1 << 63) - 1 + with assertRaises(OverflowError): + assert i64_3 & int_too_big + with assertRaises(OverflowError): + assert int_too_small & i64_3 + +def test_for_loop() -> None: + n: i64 = 0 + for i in range(i64(5 + int())): + n += i + assert n == 10 + n = 0 + for i in range(i64(5)): + n += i + assert n == 10 + n = 0 + for i in range(i64(2 + int()), 5 + int()): + n += i + assert n == 9 + n = 0 + for i in range(2, i64(5 + int())): + n += i + assert n == 9 + assert sum([x * x for x in range(i64(4 + int()))]) == 1 + 4 + 9 + +[case testI64ErrorValuesAndUndefined] +from typing import Any +import sys + +from mypy_extensions import mypyc_attr +from typing_extensions import Final + +MYPY = False +if MYPY: + from mypy_extensions import i64 + +from testutil import assertRaises + +def maybe_raise(n: i64, error: bool) -> i64: + if error: + raise ValueError() + return n + +def test_error_value() -> None: + for i in range(-1000, 1000): + assert maybe_raise(i, False) == i + with assertRaises(ValueError): + maybe_raise(0, True) + +class C: + def maybe_raise(self, n: i64, error: bool) -> i64: + if error: + raise ValueError() + return n + +def test_method_error_value() -> None: + for i in range(-1000, 1000): + assert C().maybe_raise(i, False) == i + with assertRaises(ValueError): + C().maybe_raise(0, True) + +def test_unbox_int() -> None: + for i in list(range(-1000, 1000)) + [-(1 << 63), (1 << 63) - 1]: + o: Any = i + x: i64 = i + assert x == i + y: i64 = o + assert y == i + +def test_unbox_int_fails() -> None: + o: Any = 'x' + if sys.version_info[0] == 3 and sys.version_info[1] < 10: + msg = "an integer is required (got type str)" + else: + msg = "'str' object cannot be interpreted as an integer" + with assertRaises(TypeError, msg): + x: i64 = o + o2: Any = 1 << 63 + with assertRaises(OverflowError, "int too large to convert to i64"): + y: i64 = o2 + o3: Any = -(1 << 63 + 1) + with assertRaises(OverflowError, "int too large to convert to i64"): + z: i64 = o3 + +class Uninit: + x: i64 + y: i64 = 0 + z: i64 + +class Derived(Uninit): + a: i64 = 1 + b: i64 + c: i64 = 2 + +class Derived2(Derived): + h: i64 + +def test_uninitialized_attr() -> None: + o = Uninit() + assert o.y == 0 + with assertRaises(AttributeError): + o.x + with assertRaises(AttributeError): + o.z + o.x = 1 + assert o.x == 1 + with assertRaises(AttributeError): + o.z + o.z = 2 + assert o.z == 2 + +# This is the error value, but it's also a valid normal value +MAGIC: Final = -113 + +def test_magic_value() -> None: + o = Uninit() + o.x = MAGIC + assert o.x == MAGIC + with assertRaises(AttributeError): + o.z + o.z = MAGIC + assert o.x == MAGIC + assert o.z == MAGIC + +def test_magic_value_via_any() -> None: + o: Any = Uninit() + with assertRaises(AttributeError): + o.x + with assertRaises(AttributeError): + o.z + o.x = MAGIC + assert o.x == MAGIC + with assertRaises(AttributeError): + o.z + o.z = MAGIC + assert o.z == MAGIC + +def test_magic_value_and_inheritance() -> None: + o = Derived2() + o.x = MAGIC + assert o.x == MAGIC + with assertRaises(AttributeError): + o.z + with assertRaises(AttributeError): + o.b + with assertRaises(AttributeError): + o.h + o.z = MAGIC + assert o.z == MAGIC + with assertRaises(AttributeError): + o.b + with assertRaises(AttributeError): + o.h + o.h = MAGIC + assert o.h == MAGIC + with assertRaises(AttributeError): + o.b + o.b = MAGIC + assert o.b == MAGIC + +@mypyc_attr(allow_interpreted_subclasses=True) +class MagicInit: + x: i64 = MAGIC + +def test_magic_value_as_initializer() -> None: + o = MagicInit() + assert o.x == MAGIC + +class ManyUninit: + a1: i64 + a2: i64 + a3: i64 + a4: i64 + a5: i64 + a6: i64 + a7: i64 + a8: i64 + a9: i64 + a10: i64 + a11: i64 + a12: i64 + a13: i64 + a14: i64 + a15: i64 + a16: i64 + a17: i64 + a18: i64 + a19: i64 + a20: i64 + a21: i64 + a22: i64 + a23: i64 + a24: i64 + a25: i64 + a26: i64 + a27: i64 + a28: i64 + a29: i64 + a30: i64 + a31: i64 + a32: i64 + a33: i64 + a34: i64 + a35: i64 + a36: i64 + a37: i64 + a38: i64 + a39: i64 + a40: i64 + a41: i64 + a42: i64 + a43: i64 + a44: i64 + a45: i64 + a46: i64 + a47: i64 + a48: i64 + a49: i64 + a50: i64 + a51: i64 + a52: i64 + a53: i64 + a54: i64 + a55: i64 + a56: i64 + a57: i64 + a58: i64 + a59: i64 + a60: i64 + a61: i64 + a62: i64 + a63: i64 + a64: i64 + a65: i64 + a66: i64 + a67: i64 + a68: i64 + a69: i64 + a70: i64 + a71: i64 + a72: i64 + a73: i64 + a74: i64 + a75: i64 + a76: i64 + a77: i64 + a78: i64 + a79: i64 + a80: i64 + a81: i64 + a82: i64 + a83: i64 + a84: i64 + a85: i64 + a86: i64 + a87: i64 + a88: i64 + a89: i64 + a90: i64 + a91: i64 + a92: i64 + a93: i64 + a94: i64 + a95: i64 + a96: i64 + a97: i64 + a98: i64 + a99: i64 + a100: i64 + +def test_many_uninitialized_attributes() -> None: + o = ManyUninit() + with assertRaises(AttributeError): + o.a1 + with assertRaises(AttributeError): + o.a10 + with assertRaises(AttributeError): + o.a20 + with assertRaises(AttributeError): + o.a30 + with assertRaises(AttributeError): + o.a31 + with assertRaises(AttributeError): + o.a32 + with assertRaises(AttributeError): + o.a33 + with assertRaises(AttributeError): + o.a40 + with assertRaises(AttributeError): + o.a50 + with assertRaises(AttributeError): + o.a60 + with assertRaises(AttributeError): + o.a62 + with assertRaises(AttributeError): + o.a63 + with assertRaises(AttributeError): + o.a64 + with assertRaises(AttributeError): + o.a65 + with assertRaises(AttributeError): + o.a80 + with assertRaises(AttributeError): + o.a100 + o.a30 = MAGIC + assert o.a30 == MAGIC + o.a31 = MAGIC + assert o.a31 == MAGIC + o.a32 = MAGIC + assert o.a32 == MAGIC + o.a33 = MAGIC + assert o.a33 == MAGIC + with assertRaises(AttributeError): + o.a34 + o.a62 = MAGIC + assert o.a62 == MAGIC + o.a63 = MAGIC + assert o.a63 == MAGIC + o.a64 = MAGIC + assert o.a64 == MAGIC + o.a65 = MAGIC + assert o.a65 == MAGIC + with assertRaises(AttributeError): + o.a66 + +class BaseNoBitmap: + x: int = 5 + +class DerivedBitmap(BaseNoBitmap): + # Subclass needs a bitmap, but base class doesn't have it. + y: i64 + +def test_derived_adds_bitmap() -> None: + d = DerivedBitmap() + d.x = 643 + b: BaseNoBitmap = d + assert b.x == 643 + +class Delete: + __deletable__ = ['x', 'y'] + x: i64 + y: i64 + +def test_del() -> None: + o = Delete() + o.x = MAGIC + o.y = -1 + assert o.x == MAGIC + assert o.y == -1 + del o.x + with assertRaises(AttributeError): + o.x + assert o.y == -1 + del o.y + with assertRaises(AttributeError): + o.y + o.x = 5 + assert o.x == 5 + with assertRaises(AttributeError): + o.y + del o.x + with assertRaises(AttributeError): + o.x + +[case testI64DefaultArgValues] +from typing import Any, Iterator +from typing_extensions import Final + +MAGIC: Final = -113 + +MYPY = False +if MYPY: + from mypy_extensions import i64 + +def f(x: i64, y: i64 = 5) -> i64: + return x + y + +def test_simple_default_arg() -> None: + assert f(3) == 8 + assert f(4, 9) == 13 + assert f(5, MAGIC) == -108 + for i in range(-1000, 1000): + assert f(1, i) == 1 + i + f2: Any = f + assert f2(3) == 8 + assert f2(4, 9) == 13 + assert f2(5, MAGIC) == -108 + +def g(a: i64, b: i64 = 1, c: int = 2, d: i64 = 3) -> i64: + return a + b + c + d + +def test_two_default_args() -> None: + assert g(10) == 16 + assert g(10, 2) == 17 + assert g(10, 2, 3) == 18 + assert g(10, 2, 3, 4) == 19 + g2: Any = g + assert g2(10) == 16 + assert g2(10, 2) == 17 + assert g2(10, 2, 3) == 18 + assert g2(10, 2, 3, 4) == 19 + +class C: + def __init__(self) -> None: + self.i: i64 = 1 + + def m(self, a: i64, b: i64 = 1, c: int = 2, d: i64 = 3) -> i64: + return self.i + a + b + c + d + +class D(C): + def m(self, a: i64, b: i64 = 2, c: int = 3, d: i64 = 4) -> i64: + return self.i + a + b + c + d + + def mm(self, a: i64 = 2, b: i64 = 1) -> i64: + return self.i + a + b + + @staticmethod + def s(a: i64 = 2, b: i64 = 1) -> i64: + return a + b + + @classmethod + def c(cls, a: i64 = 2, b: i64 = 3) -> i64: + assert cls is D + return a + b + +def test_method_default_args() -> None: + a = [C(), D()] + assert a[0].m(4) == 11 + d = D() + assert d.mm() == 4 + assert d.mm(5) == 7 + assert d.mm(MAGIC) == MAGIC + 2 + assert d.mm(b=5) == 8 + assert D.mm(d) == 4 + assert D.mm(d, 6) == 8 + assert D.mm(d, MAGIC) == MAGIC + 2 + assert D.mm(d, b=6) == 9 + dd: Any = d + assert dd.mm() == 4 + assert dd.mm(5) == 7 + assert dd.mm(MAGIC) == MAGIC + 2 + assert dd.mm(b=5) == 8 + +def test_static_method_default_args() -> None: + d = D() + assert d.s() == 3 + assert d.s(5) == 6 + assert d.s(MAGIC) == MAGIC + 1 + assert d.s(5, 6) == 11 + assert D.s() == 3 + assert D.s(5) == 6 + assert D.s(MAGIC) == MAGIC + 1 + assert D.s(5, 6) == 11 + dd: Any = d + assert dd.s() == 3 + assert dd.s(5) == 6 + assert dd.s(MAGIC) == MAGIC + 1 + assert dd.s(5, 6) == 11 + +def test_class_method_default_args() -> None: + d = D() + assert d.c() == 5 + assert d.c(5) == 8 + assert d.c(MAGIC) == MAGIC + 3 + assert d.c(b=5) == 7 + assert D.c() == 5 + assert D.c(5) == 8 + assert D.c(MAGIC) == MAGIC + 3 + assert D.c(b=5) == 7 + dd: Any = d + assert dd.c() == 5 + assert dd.c(5) == 8 + assert dd.c(MAGIC) == MAGIC + 3 + assert dd.c(b=5) == 7 + +class Init: + def __init__(self, x: i64 = 2, y: i64 = 5) -> None: + self.x = x + self.y = y + +def test_init_default_args() -> None: + o = Init() + assert o.x == 2 + assert o.y == 5 + o = Init(7, 8) + assert o.x == 7 + assert o.y == 8 + o = Init(4) + assert o.x == 4 + assert o.y == 5 + o = Init(MAGIC, MAGIC) + assert o.x == MAGIC + assert o.y == MAGIC + o = Init(3, MAGIC) + assert o.x == 3 + assert o.y == MAGIC + o = Init(MAGIC, 11) + assert o.x == MAGIC + assert o.y == 11 + o = Init(MAGIC) + assert o.x == MAGIC + assert o.y == 5 + o = Init(y=MAGIC) + assert o.x == 2 + assert o.y == MAGIC + +def kw_only(*, a: i64 = 1, b: int = 2, c: i64 = 3) -> i64: + return a + b + c * 2 + +def test_kw_only_default_args() -> None: + assert kw_only() == 9 + assert kw_only(a=2) == 10 + assert kw_only(b=4) == 11 + assert kw_only(c=11) == 25 + assert kw_only(a=2, c=4) == 12 + assert kw_only(c=4, a=2) == 12 + kw_only2: Any = kw_only + assert kw_only2() == 9 + assert kw_only2(a=2) == 10 + assert kw_only2(b=4) == 11 + assert kw_only2(c=11) == 25 + assert kw_only2(a=2, c=4) == 12 + assert kw_only2(c=4, a=2) == 12 + +def many_args( + a1: i64 = 0, + a2: i64 = 1, + a3: i64 = 2, + a4: i64 = 3, + a5: i64 = 4, + a6: i64 = 5, + a7: i64 = 6, + a8: i64 = 7, + a9: i64 = 8, + a10: i64 = 9, + a11: i64 = 10, + a12: i64 = 11, + a13: i64 = 12, + a14: i64 = 13, + a15: i64 = 14, + a16: i64 = 15, + a17: i64 = 16, + a18: i64 = 17, + a19: i64 = 18, + a20: i64 = 19, + a21: i64 = 20, + a22: i64 = 21, + a23: i64 = 22, + a24: i64 = 23, + a25: i64 = 24, + a26: i64 = 25, + a27: i64 = 26, + a28: i64 = 27, + a29: i64 = 28, + a30: i64 = 29, + a31: i64 = 30, + a32: i64 = 31, + a33: i64 = 32, + a34: i64 = 33, +) -> i64: + return a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8 + a9 + a10 + a11 + a12 + a13 + a14 + a15 + a16 + a17 + a18 + a19 + a20 + a21 + a22 + a23 + a24 + a25 + a26 + a27 + a28 + a29 + a30 + a31 + a32 + a33 + a34 + +def test_many_args() -> None: + assert many_args() == 561 + assert many_args(a1=100) == 661 + assert many_args(a2=101) == 661 + assert many_args(a15=114) == 661 + assert many_args(a31=130) == 661 + assert many_args(a32=131) == 661 + assert many_args(a33=232) == 761 + assert many_args(a34=333) == 861 + assert many_args(a1=100, a33=232) == 861 + f: Any = many_args + assert f() == 561 + assert f(a1=100) == 661 + assert f(a2=101) == 661 + assert f(a15=114) == 661 + assert f(a31=130) == 661 + assert f(a32=131) == 661 + assert f(a33=232) == 761 + assert f(a34=333) == 861 + assert f(a1=100, a33=232) == 861 + +def test_nested_function_defaults() -> None: + a: i64 = 1 + + def nested(x: i64 = 2, y: i64 = 3) -> i64: + return a + x + y + + assert nested() == 6 + assert nested(3) == 7 + assert nested(y=5) == 8 + assert nested(MAGIC) == MAGIC + 4 + a = 11 + assert nested() == 16 + + +def test_nested_function_defaults_via_any() -> None: + a: i64 = 1 + + def nested_native(x: i64 = 2, y: i64 = 3) -> i64: + return a + x + y + + nested: Any = nested_native + + assert nested() == 6 + assert nested(3) == 7 + assert nested(y=5) == 8 + assert nested(MAGIC) == MAGIC + 4 + a = 11 + assert nested() == 16 + +def gen(x: i64 = 1, y: i64 = 2) -> Iterator[i64]: + yield x + y + +def test_generator() -> None: + g = gen() + assert next(g) == 3 + g = gen(2) + assert next(g) == 4 + g = gen(2, 3) + assert next(g) == 5 + a: Any = gen + g = a() + assert next(g) == 3 + g = a(2) + assert next(g) == 4 + g = a(2, 3) + assert next(g) == 5 + +def magic_default(x: i64 = MAGIC) -> i64: + return x + +def test_magic_default() -> None: + assert magic_default() == MAGIC + assert magic_default(1) == 1 + assert magic_default(MAGIC) == MAGIC + a: Any = magic_default + assert a() == MAGIC + assert a(1) == 1 + assert a(MAGIC) == MAGIC + +[case testI64UndefinedLocal] +from typing_extensions import Final + +MYPY = False +if MYPY: + from mypy_extensions import i64, i32 + +from testutil import assertRaises + +MAGIC: Final = -113 + + +def test_conditionally_defined_local() -> None: + x = not int() + if x: + y: i64 = 5 + z: i32 = 6 + assert y == 5 + assert z == 6 + +def test_conditionally_undefined_local() -> None: + x = int() + if x: + y: i64 = 5 + z: i32 = 6 + else: + ok: i64 = 7 + assert ok == 7 + try: + print(y) + except NameError as e: + assert str(e) == 'local variable "y" referenced before assignment' + else: + assert False + try: + print(z) + except NameError as e: + assert str(e) == 'local variable "z" referenced before assignment' + else: + assert False + +def test_assign_error_value_conditionally() -> None: + x = int() + if not x: + y: i64 = MAGIC + z: i32 = MAGIC + assert y == MAGIC + assert z == MAGIC + +def test_many_locals() -> None: + x = int() + if x: + a0: i64 = 0 + a1: i64 = 1 + a2: i64 = 2 + a3: i64 = 3 + a4: i64 = 4 + a5: i64 = 5 + a6: i64 = 6 + a7: i64 = 7 + a8: i64 = 8 + a9: i64 = 9 + a10: i64 = 10 + a11: i64 = 11 + a12: i64 = 12 + a13: i64 = 13 + a14: i64 = 14 + a15: i64 = 15 + a16: i64 = 16 + a17: i64 = 17 + a18: i64 = 18 + a19: i64 = 19 + a20: i64 = 20 + a21: i64 = 21 + a22: i64 = 22 + a23: i64 = 23 + a24: i64 = 24 + a25: i64 = 25 + a26: i64 = 26 + a27: i64 = 27 + a28: i64 = 28 + a29: i64 = 29 + a30: i64 = 30 + a31: i64 = 31 + a32: i64 = 32 + a33: i64 = 33 + with assertRaises(NameError): + print(a0) + with assertRaises(NameError): + print(a31) + with assertRaises(NameError): + print(a32) + with assertRaises(NameError): + print(a33) + a0 = 5 + assert a0 == 5 + with assertRaises(NameError): + print(a31) + with assertRaises(NameError): + print(a32) + with assertRaises(NameError): + print(a33) + a32 = 55 + assert a0 == 5 + assert a32 == 55 + with assertRaises(NameError): + print(a31) + with assertRaises(NameError): + print(a33) + a31 = 10 + a33 = 20 + assert a0 == 5 + assert a31 == 10 + assert a32 == 55 + assert a33 == 20 + +[case testI64GlueMethods] +from typing_extensions import Final + +MYPY = False +if MYPY: + from mypy_extensions import i64 + +MAGIC: Final = -113 + +class Base: + def foo(self) -> i64: + return 5 + + def bar(self, x: i64 = 2) -> i64: + return x + 1 + + def hoho(self, x: i64) -> i64: + return x - 1 + +class Derived(Base): + def foo(self, x: i64 = 5) -> i64: + return x + 10 + + def bar(self, x: i64 = 3, y: i64 = 20) -> i64: + return x + y + 2 + + def hoho(self, x: i64 = 7) -> i64: + return x - 2 + +def test_derived_adds_bitmap() -> None: + b: Base = Derived() + assert b.foo() == 15 + +def test_derived_adds_another_default_arg() -> None: + b: Base = Derived() + assert b.bar() == 25 + assert b.bar(1) == 23 + assert b.bar(MAGIC) == MAGIC + 22 + +def test_derived_switches_arg_to_have_default() -> None: + b: Base = Derived() + assert b.hoho(5) == 3 + assert b.hoho(MAGIC) == MAGIC - 2 diff --git a/mypyc/test/test_commandline.py b/mypyc/test/test_commandline.py index 1822cf13fe42..aafe1e4adc1b 100644 --- a/mypyc/test/test_commandline.py +++ b/mypyc/test/test_commandline.py @@ -43,6 +43,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: with open(program_path, "w") as f: f.write(text) + env = os.environ.copy() + env["PYTHONPATH"] = base_path + out = b"" try: # Compile program @@ -51,6 +54,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd="tmp", + env=env, ) if "ErrorOutput" in testcase.name or cmd.returncode != 0: out += cmd.stdout diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 5be1e61cba8d..d7dcf3be532b 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -9,6 +9,7 @@ from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature, RuntimeArg from mypyc.ir.ops import ( + ERR_NEVER, Assign, AssignMulti, BasicBlock, @@ -103,7 +104,13 @@ def add_local(name: str, rtype: RType) -> Register: "tt", RTuple([RTuple([int_rprimitive, bool_rprimitive]), bool_rprimitive]) ) ir = ClassIR("A", "mod") - ir.attributes = {"x": bool_rprimitive, "y": int_rprimitive} + ir.attributes = { + "x": bool_rprimitive, + "y": int_rprimitive, + "i1": int64_rprimitive, + "i2": int32_rprimitive, + } + ir.bitmap_attrs = ["i1", "i2"] compute_vtable(ir) ir.mro = [ir] self.r = add_local("r", RInstance(ir)) @@ -397,6 +404,16 @@ def test_get_attr_merged(self) -> None: skip_next=True, ) + def test_get_attr_with_bitmap(self) -> None: + self.assert_emit( + GetAttr(self.r, "i1", 1), + """cpy_r_r0 = ((mod___AObject *)cpy_r_r)->_i1; + if (unlikely(cpy_r_r0 == -113) && !(((mod___AObject *)cpy_r_r)->bitmap & 1)) { + PyErr_SetString(PyExc_AttributeError, "attribute 'i1' of 'A' undefined"); + } + """, + ) + def test_set_attr(self) -> None: self.assert_emit( SetAttr(self.r, "y", self.m, 1), @@ -416,6 +433,62 @@ def test_set_attr_non_refcounted(self) -> None: """, ) + def test_set_attr_no_error(self) -> None: + op = SetAttr(self.r, "y", self.m, 1) + op.error_kind = ERR_NEVER + self.assert_emit( + op, + """if (((mod___AObject *)cpy_r_r)->_y != CPY_INT_TAG) { + CPyTagged_DECREF(((mod___AObject *)cpy_r_r)->_y); + } + ((mod___AObject *)cpy_r_r)->_y = cpy_r_m; + """, + ) + + def test_set_attr_non_refcounted_no_error(self) -> None: + op = SetAttr(self.r, "x", self.b, 1) + op.error_kind = ERR_NEVER + self.assert_emit( + op, + """((mod___AObject *)cpy_r_r)->_x = cpy_r_b; + """, + ) + + def test_set_attr_with_bitmap(self) -> None: + # For some rtypes the error value overlaps a valid value, so we need + # to use a separate bitmap to track defined attributes. + self.assert_emit( + SetAttr(self.r, "i1", self.i64, 1), + """if (unlikely(cpy_r_i64 == -113)) { + ((mod___AObject *)cpy_r_r)->bitmap |= 1; + } + ((mod___AObject *)cpy_r_r)->_i1 = cpy_r_i64; + cpy_r_r0 = 1; + """, + ) + self.assert_emit( + SetAttr(self.r, "i2", self.i32, 1), + """if (unlikely(cpy_r_i32 == -113)) { + ((mod___AObject *)cpy_r_r)->bitmap |= 2; + } + ((mod___AObject *)cpy_r_r)->_i2 = cpy_r_i32; + cpy_r_r0 = 1; + """, + ) + + def test_set_attr_init_with_bitmap(self) -> None: + op = SetAttr(self.r, "i1", self.i64, 1) + op.is_init = True + self.assert_emit( + op, + """if (unlikely(cpy_r_i64 == -113)) { + ((mod___AObject *)cpy_r_r)->bitmap |= 1; + } + ((mod___AObject *)cpy_r_r)->_i1 = cpy_r_i64; + cpy_r_r0 = 1; + """, + ) + def test_dict_get_item(self) -> None: self.assert_emit( CallC( diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index ba8014116e8a..00a8c074da87 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -37,12 +37,15 @@ "irbuild-generics.test", "irbuild-try.test", "irbuild-strip-asserts.test", + "irbuild-i64.test", + "irbuild-i32.test", "irbuild-vectorcall.test", "irbuild-unreachable.test", "irbuild-isinstance.test", "irbuild-dunders.test", "irbuild-singledispatch.test", "irbuild-constant-fold.test", + "irbuild-glue-methods.test", ] diff --git a/mypyc/test/test_ircheck.py b/mypyc/test/test_ircheck.py index 30ddd39fef0d..008963642272 100644 --- a/mypyc/test/test_ircheck.py +++ b/mypyc/test/test_ircheck.py @@ -5,7 +5,17 @@ from mypyc.analysis.ircheck import FnError, can_coerce_to, check_func_ir from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncDecl, FuncIR, FuncSignature -from mypyc.ir.ops import Assign, BasicBlock, Goto, Integer, LoadLiteral, Op, Register, Return +from mypyc.ir.ops import ( + Assign, + BasicBlock, + Goto, + Integer, + LoadAddress, + LoadLiteral, + Op, + Register, + Return, +) from mypyc.ir.pprint import format_func from mypyc.ir.rtypes import ( RInstance, @@ -16,6 +26,7 @@ int64_rprimitive, none_rprimitive, object_rprimitive, + pointer_rprimitive, str_rprimitive, ) @@ -88,7 +99,7 @@ def test_invalid_register_source(self) -> None: ret = Return(value=Register(type=none_rprimitive, name="r1")) block = self.basic_block([ret]) fn = FuncIR(decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block]) - assert_has_error(fn, FnError(source=ret, desc="Invalid op reference to register r1")) + assert_has_error(fn, FnError(source=ret, desc="Invalid op reference to register 'r1'")) def test_invalid_op_source(self) -> None: ret = Return(value=LoadLiteral(value="foo", rtype=str_rprimitive)) @@ -170,3 +181,19 @@ def test_pprint(self) -> None: " goto L1", " ERR: Invalid control operation target: 1", ] + + def test_load_address_declares_register(self) -> None: + rx = Register(str_rprimitive, "x") + ry = Register(pointer_rprimitive, "y") + load_addr = LoadAddress(pointer_rprimitive, rx) + assert_no_errors( + FuncIR( + decl=self.func_decl(name="func_1"), + arg_regs=[], + blocks=[ + self.basic_block( + ops=[load_addr, Assign(ry, load_addr), Return(value=NONE_VALUE)] + ) + ], + ) + ) diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 62168ff4bb00..351caf7c93ed 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -10,11 +10,12 @@ import shutil import subprocess import sys +import time from typing import Any, Iterator, cast from mypy import build from mypy.errors import CompileError -from mypy.options import Options +from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase from mypy.test.helpers import assert_module_equivalence, perform_file_operations @@ -38,6 +39,8 @@ "run-misc.test", "run-functions.test", "run-integers.test", + "run-i64.test", + "run-i32.test", "run-floats.test", "run-bools.test", "run-strings.test", @@ -167,6 +170,12 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None: # new by distutils, shift the mtime of all of the # generated artifacts back by a second. fudge_dir_mtimes(WORKDIR, -1) + # On Ubuntu, changing the mtime doesn't work reliably. As + # a workaround, sleep. + # + # TODO: Figure out a better approach, since this slows down tests. + if sys.platform == "linux": + time.sleep(1.0) step += 1 with chdir_manager(".."): @@ -183,7 +192,9 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> options.python_version = sys.version_info[:2] options.export_types = True options.preserve_asts = True + options.allow_empty_bodies = True options.incremental = self.separate + options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] # Avoid checking modules/packages named 'unchecked', to provide a way # to test interacting with code we don't have types for. diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py index dc771b00551d..8339889fa9f5 100644 --- a/mypyc/test/testutil.py +++ b/mypyc/test/testutil.py @@ -105,11 +105,13 @@ def build_ir_for_single_file2( compiler_options = compiler_options or CompilerOptions(capi_version=(3, 5)) options = Options() options.show_traceback = True + options.hide_error_codes = True options.use_builtins_fixtures = True options.strict_optional = True options.python_version = (3, 6) options.export_types = True options.preserve_asts = True + options.allow_empty_bodies = True options.per_module_options["__main__"] = {"mypyc": True} source = build.BuildSource("main", "__main__", program_text) diff --git a/mypyc/transform/exceptions.py b/mypyc/transform/exceptions.py index 3cfe6e5d3bd5..cc638142c397 100644 --- a/mypyc/transform/exceptions.py +++ b/mypyc/transform/exceptions.py @@ -23,10 +23,12 @@ Branch, CallC, ComparisonOp, + GetAttr, Integer, LoadErrorValue, RegisterOp, Return, + SetAttr, Value, ) from mypyc.ir.rtypes import bool_rprimitive @@ -40,6 +42,7 @@ def insert_exception_handling(ir: FuncIR) -> None: # block. The block just returns an error value. error_label = None for block in ir.blocks: + adjust_error_kinds(block) can_raise = any(op.can_raise() for op in block.ops) if can_raise: error_label = add_handler_block(ir) @@ -103,7 +106,11 @@ def split_blocks_at_errors( new_block2 = BasicBlock() new_blocks.append(new_block2) branch = Branch( - comp, true_label=new_block2, false_label=new_block, op=Branch.BOOL + comp, + true_label=new_block2, + false_label=new_block, + op=Branch.BOOL, + rare=True, ) cur_block.ops.append(branch) cur_block = new_block2 @@ -141,3 +148,18 @@ def primitive_call(desc: CFunctionDescription, args: list[Value], line: int) -> desc.error_kind, line, ) + + +def adjust_error_kinds(block: BasicBlock) -> None: + """Infer more precise error_kind attributes for ops. + + We have access here to more information than what was available + when the IR was initially built. + """ + for op in block.ops: + if isinstance(op, GetAttr): + if op.class_type.class_ir.is_always_defined(op.attr): + op.error_kind = ERR_NEVER + if isinstance(op, SetAttr): + if op.class_type.class_ir.is_always_defined(op.attr): + op.error_kind = ERR_NEVER diff --git a/mypyc/transform/uninit.py b/mypyc/transform/uninit.py index 0fa65be90295..041dd2545dff 100644 --- a/mypyc/transform/uninit.py +++ b/mypyc/transform/uninit.py @@ -3,11 +3,15 @@ from __future__ import annotations from mypyc.analysis.dataflow import AnalysisDict, analyze_must_defined_regs, cleanup_cfg, get_cfg +from mypyc.common import BITMAP_BITS from mypyc.ir.func_ir import FuncIR, all_values from mypyc.ir.ops import ( Assign, BasicBlock, Branch, + ComparisonOp, + Integer, + IntOp, LoadAddress, LoadErrorValue, Op, @@ -16,6 +20,7 @@ Unreachable, Value, ) +from mypyc.ir.rtypes import bitmap_rprimitive, is_fixed_width_rtype def insert_uninit_checks(ir: FuncIR) -> None: @@ -38,6 +43,8 @@ def split_blocks_at_uninits( init_registers = [] init_registers_set = set() + bitmap_registers: list[Register] = [] # Init status bitmaps + bitmap_backed: list[Register] = [] # These use bitmaps to track init status # First split blocks on ops that may raise. for block in blocks: @@ -70,15 +77,28 @@ def split_blocks_at_uninits( init_registers.append(src) init_registers_set.add(src) - cur_block.ops.append( - Branch( + if not is_fixed_width_rtype(src.type): + cur_block.ops.append( + Branch( + src, + true_label=error_block, + false_label=new_block, + op=Branch.IS_ERROR, + line=op.line, + ) + ) + else: + # We need to use bitmap for this one. + check_for_uninit_using_bitmap( + cur_block.ops, src, - true_label=error_block, - false_label=new_block, - op=Branch.IS_ERROR, - line=op.line, + bitmap_registers, + bitmap_backed, + error_block, + new_block, + op.line, ) - ) + raise_std = RaiseStandardError( RaiseStandardError.UNBOUND_LOCAL_ERROR, f'local variable "{src.name}" referenced before assignment', @@ -89,12 +109,82 @@ def split_blocks_at_uninits( cur_block = new_block cur_block.ops.append(op) + if bitmap_backed: + update_register_assignments_to_set_bitmap(new_blocks, bitmap_registers, bitmap_backed) + if init_registers: new_ops: list[Op] = [] for reg in init_registers: err = LoadErrorValue(reg.type, undefines=True) new_ops.append(err) new_ops.append(Assign(reg, err)) + for reg in bitmap_registers: + new_ops.append(Assign(reg, Integer(0, bitmap_rprimitive))) new_blocks[0].ops[0:0] = new_ops return new_blocks + + +def check_for_uninit_using_bitmap( + ops: list[Op], + src: Register, + bitmap_registers: list[Register], + bitmap_backed: list[Register], + error_block: BasicBlock, + ok_block: BasicBlock, + line: int, +) -> None: + """Check if src is defined using a bitmap. + + Modifies ops, bitmap_registers and bitmap_backed. + """ + if src not in bitmap_backed: + # Set up a new bitmap backed register. + bitmap_backed.append(src) + n = (len(bitmap_backed) - 1) // BITMAP_BITS + if len(bitmap_registers) <= n: + bitmap_registers.append(Register(bitmap_rprimitive, f"__locals_bitmap{n}")) + + index = bitmap_backed.index(src) + masked = IntOp( + bitmap_rprimitive, + bitmap_registers[index // BITMAP_BITS], + Integer(1 << (index & (BITMAP_BITS - 1)), bitmap_rprimitive), + IntOp.AND, + line, + ) + ops.append(masked) + chk = ComparisonOp(masked, Integer(0, bitmap_rprimitive), ComparisonOp.EQ) + ops.append(chk) + ops.append(Branch(chk, error_block, ok_block, Branch.BOOL)) + + +def update_register_assignments_to_set_bitmap( + blocks: list[BasicBlock], bitmap_registers: list[Register], bitmap_backed: list[Register] +) -> None: + """Update some assignments to registers to also set a bit in a bitmap. + + The bitmaps are used to track if a local variable has been assigned to. + + Modifies blocks. + """ + for block in blocks: + if any(isinstance(op, Assign) and op.dest in bitmap_backed for op in block.ops): + new_ops: list[Op] = [] + for op in block.ops: + if isinstance(op, Assign) and op.dest in bitmap_backed: + index = bitmap_backed.index(op.dest) + new_ops.append(op) + reg = bitmap_registers[index // BITMAP_BITS] + new = IntOp( + bitmap_rprimitive, + reg, + Integer(1 << (index & (BITMAP_BITS - 1)), bitmap_rprimitive), + IntOp.OR, + op.line, + ) + new_ops.append(new) + new_ops.append(Assign(reg, new)) + else: + new_ops.append(op) + block.ops = new_ops diff --git a/pyproject.toml b/pyproject.toml index 95f65599a130..1348b9463639 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,27 @@ [build-system] requires = [ + # NOTE: this needs to be kept in sync with mypy-requirements.txt + # and build-requirements.txt, because those are both needed for + # self-typechecking :/ "setuptools >= 40.6.2", "wheel >= 0.30.0", + # the following is from mypy-requirements.txt + "typing_extensions>=3.10", + "mypy_extensions>=0.4.3", + "typed_ast>=1.4.0,<2; python_version<'3.8'", + "tomli>=1.1.0; python_version<'3.11'", + # the following is from build-requirements.txt + "types-psutil", + "types-setuptools", + "types-typed-ast>=1.5.8,<1.6.0", ] build-backend = "setuptools.build_meta" [tool.black] line-length = 99 -target-version = ['py37'] +target-version = ["py37", "py38", "py39", "py310", "py311"] skip-magic-trailing-comma = true -extend-exclude = ''' +force-exclude = ''' ^/mypy/typeshed| ^/mypyc/test-data| ^/test-data @@ -21,8 +33,8 @@ line_length = 99 combine_as_imports = true skip_gitignore = true extra_standard_library = ["typing_extensions"] -skip = [ - "mypy/typeshed", - "mypyc/test-data", - "test-data", +skip_glob = [ + "mypy/typeshed/*", + "mypyc/test-data/*", + "test-data/*", ] diff --git a/runtests.py b/runtests.py index c41f1db7e40f..be4ad4add08a 100755 --- a/runtests.py +++ b/runtests.py @@ -8,8 +8,6 @@ # Slow test suites CMDLINE = "PythonCmdline" -SAMPLES = "SamplesSuite" -TYPESHED = "TypeshedSuite" PEP561 = "PEP561Suite" EVALUATION = "PythonEvaluation" DAEMON = "testdaemon" @@ -24,8 +22,6 @@ ALL_NON_FAST = [ CMDLINE, - SAMPLES, - TYPESHED, PEP561, EVALUATION, DAEMON, @@ -71,12 +67,10 @@ "pytest", "-q", "-k", - " or ".join([SAMPLES, TYPESHED, DAEMON, MYPYC_EXTERNAL, MYPYC_COMMAND_LINE, ERROR_STREAM]), + " or ".join([DAEMON, MYPYC_EXTERNAL, MYPYC_COMMAND_LINE, ERROR_STREAM]), ], # Test cases that might take minutes to run "pytest-extra": ["pytest", "-q", "-k", " or ".join(PYTEST_OPT_IN)], - # Test cases to run in typeshed CI - "typeshed-ci": ["pytest", "-q", "-k", " or ".join([CMDLINE, EVALUATION, SAMPLES, TYPESHED])], # Mypyc tests that aren't run by default, since they are slow and rarely # fail for commits that don't touch mypyc "mypyc-extra": ["pytest", "-q", "-k", " or ".join(MYPYC_OPT_IN)], @@ -85,7 +79,7 @@ # Stop run immediately if these commands fail FAST_FAIL = ["self", "lint"] -EXTRA_COMMANDS = ("pytest-extra", "mypyc-extra", "typeshed-ci") +EXTRA_COMMANDS = ("pytest-extra", "mypyc-extra") DEFAULT_COMMANDS = [cmd for cmd in cmds if cmd not in EXTRA_COMMANDS] assert all(cmd in cmds for cmd in FAST_FAIL) diff --git a/setup.cfg b/setup.cfg index 326b5bb53e96..511f794474e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,10 +19,6 @@ exclude = # Sphinx configuration is irrelevant docs/source/conf.py, mypyc/doc/conf.py, - # conflicting styles - misc/*, - # conflicting styles - scripts/*, # tests have more relaxed styling requirements # fixtures have their own .pyi-specific configuration test-data/*, @@ -42,9 +38,8 @@ exclude = # B007: Loop control variable not used within the loop body. # B011: Don't use assert False # B023: Function definition does not bind loop variable -# F821: Name not defined (generates false positives with error codes) # E741: Ambiguous variable name -extend-ignore = E203,E501,W601,E402,B006,B007,B011,B023,F821,E741 +extend-ignore = E203,E501,W601,E402,B006,B007,B011,B023,E741 [coverage:run] branch = true diff --git a/setup.py b/setup.py index a8c86ff663a3..669e0cc4b615 100644 --- a/setup.py +++ b/setup.py @@ -79,8 +79,8 @@ def run(self): USE_MYPYC = False # To compile with mypyc, a mypyc checkout must be present on the PYTHONPATH -if len(sys.argv) > 1 and sys.argv[1] == "--use-mypyc": - sys.argv.pop(1) +if len(sys.argv) > 1 and "--use-mypyc" in sys.argv: + sys.argv.remove("--use-mypyc") USE_MYPYC = True if os.getenv("MYPY_USE_MYPYC", None) == "1": USE_MYPYC = True @@ -180,6 +180,7 @@ def run(self): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development", "Typing :: Typed", ] @@ -220,6 +221,7 @@ def run(self): "dmypy": "psutil >= 4.0", "python2": "typed_ast >= 1.4.0, < 2", "reports": "lxml", + "install-types": "pip", }, python_requires=">=3.7", include_package_data=True, diff --git a/test-data/unit/README.md b/test-data/unit/README.md index 39ab918faddb..6cf0b1bb26cf 100644 --- a/test-data/unit/README.md +++ b/test-data/unit/README.md @@ -176,6 +176,10 @@ full builtins and library stubs instead of minimal ones. Run them using Note that running more processes than logical cores is likely to significantly decrease performance. +To run tests with coverage: + + python3 -m pytest --cov mypy --cov-config setup.cfg --cov-report=term-missing:skip-covered --cov-report=html + Debugging --------- diff --git a/test-data/unit/check-abstract.test b/test-data/unit/check-abstract.test index e384cb89120b..f67d9859397e 100644 --- a/test-data/unit/check-abstract.test +++ b/test-data/unit/check-abstract.test @@ -314,8 +314,8 @@ class B(A): # E: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" \ # N: This violates the Liskov substitution principle \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides - pass - def g(self, x: int) -> int: pass + return 0 + def g(self, x: int) -> int: return 0 [out] [case testImplementingAbstractMethodWithMultipleBaseClasses] @@ -328,13 +328,13 @@ class J(metaclass=ABCMeta): @abstractmethod def g(self, x: str) -> str: pass class A(I, J): - def f(self, x: str) -> int: pass \ + def f(self, x: str) -> int: return 0 \ # E: Argument 1 of "f" is incompatible with supertype "I"; supertype defines the argument type as "int" \ # N: This violates the Liskov substitution principle \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides - def g(self, x: str) -> int: pass \ + def g(self, x: str) -> int: return 0 \ # E: Return type "int" of "g" incompatible with return type "str" in supertype "J" - def h(self) -> int: pass # Not related to any base class + def h(self) -> int: return 0 # Not related to any base class [out] [case testImplementingAbstractMethodWithExtension] @@ -345,7 +345,7 @@ class J(metaclass=ABCMeta): def f(self, x: int) -> int: pass class I(J): pass class A(I): - def f(self, x: str) -> int: pass \ + def f(self, x: str) -> int: return 0 \ # E: Argument 1 of "f" is incompatible with supertype "J"; supertype defines the argument type as "int" \ # N: This violates the Liskov substitution principle \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides @@ -376,11 +376,11 @@ class I(metaclass=ABCMeta): def h(self, a: 'I') -> A: pass class A(I): def h(self, a: 'A') -> 'I': # Fail - pass + return A() def f(self, a: 'I') -> 'I': - pass + return A() def g(self, a: 'A') -> 'A': - pass + return A() [out] main:11: error: Argument 1 of "h" is incompatible with supertype "I"; supertype defines the argument type as "I" main:11: note: This violates the Liskov substitution principle @@ -672,7 +672,7 @@ class A(metaclass=ABCMeta): def __gt__(self, other: 'A') -> int: pass [case testAbstractOperatorMethods2] -import typing +from typing import cast, Any from abc import abstractmethod, ABCMeta class A(metaclass=ABCMeta): @abstractmethod @@ -681,7 +681,8 @@ class B: @abstractmethod def __add__(self, other: 'A') -> int: pass class C: - def __add__(self, other: int) -> B: pass + def __add__(self, other: int) -> B: + return cast(Any, None) [out] [case testAbstractClassWithAnyBase] @@ -761,7 +762,7 @@ class A(metaclass=ABCMeta): def x(self) -> int: pass class B(A): @property - def x(self) -> int: pass + def x(self) -> int: return 0 b = B() b.x() # E: "int" not callable [builtins fixtures/property.pyi] @@ -775,7 +776,7 @@ class A(metaclass=ABCMeta): def x(self, v: int) -> None: pass class B(A): @property - def x(self) -> int: pass + def x(self) -> int: return 0 @x.setter def x(self, v: int) -> None: pass b = B() @@ -789,7 +790,7 @@ class A(metaclass=ABCMeta): def x(self) -> int: pass class B(A): @property - def x(self) -> str: pass # E: Return type "str" of "x" incompatible with return type "int" in supertype "A" + def x(self) -> str: return "no" # E: Signature of "x" incompatible with supertype "A" b = B() b.x() # E: "str" not callable [builtins fixtures/property.pyi] @@ -850,7 +851,7 @@ class A(metaclass=ABCMeta): def x(self, v: int) -> None: pass class B(A): @property # E - def x(self) -> int: pass + def x(self) -> int: return 0 b = B() b.x.y # E [builtins fixtures/property.pyi] @@ -906,7 +907,7 @@ class C(Mixin, A): class A: @property def foo(cls) -> str: - pass + return "yes" class Mixin: foo = "foo" class C(Mixin, A): @@ -922,7 +923,7 @@ class Y(X): class A: @property def foo(cls) -> X: - pass + return X() class Mixin: foo = Y() class C(Mixin, A): @@ -934,7 +935,7 @@ class C(Mixin, A): class A: @property def foo(cls) -> str: - pass + return "no" class Mixin: foo = "foo" class C(A, Mixin): # E: Definition of "foo" in base class "A" is incompatible with definition in base class "Mixin" @@ -1024,7 +1025,7 @@ from abc import abstractmethod, ABCMeta from typing import Type, TypeVar T = TypeVar("T") -def deco(cls: Type[T]) -> Type[T]: ... +def deco(cls: Type[T]) -> Type[T]: return cls @deco class A(metaclass=ABCMeta): @@ -1050,3 +1051,579 @@ b: B b.x = 1 # E: Property "x" defined in "B" is read-only b.y = 1 [builtins fixtures/property.pyi] + + +-- Treatment of empty bodies in ABCs and protocols +-- ----------------------------------------------- + +[case testEmptyBodyProhibitedFunction] +# flags: --strict-optional +from typing import overload, Union + +def func1(x: str) -> int: pass # E: Missing return statement +def func2(x: str) -> int: ... # E: Missing return statement +def func3(x: str) -> int: # E: Missing return statement + """Some function.""" + +@overload +def func4(x: int) -> int: ... +@overload +def func4(x: str) -> str: ... +def func4(x: Union[int, str]) -> Union[int, str]: # E: Missing return statement + pass + +@overload +def func5(x: int) -> int: ... +@overload +def func5(x: str) -> str: ... +def func5(x: Union[int, str]) -> Union[int, str]: # E: Missing return statement + """Some function.""" + +[case testEmptyBodyProhibitedMethodNonAbstract] +# flags: --strict-optional +from typing import overload, Union + +class A: + def func1(self, x: str) -> int: pass # E: Missing return statement + def func2(self, x: str) -> int: ... # E: Missing return statement + def func3(self, x: str) -> int: # E: Missing return statement + """Some function.""" + +class B: + @classmethod + def func1(cls, x: str) -> int: pass # E: Missing return statement + @classmethod + def func2(cls, x: str) -> int: ... # E: Missing return statement + @classmethod + def func3(cls, x: str) -> int: # E: Missing return statement + """Some function.""" + +class C: + @overload + def func4(self, x: int) -> int: ... + @overload + def func4(self, x: str) -> str: ... + def func4(self, x: Union[int, str]) -> Union[int, str]: # E: Missing return statement + pass + + @overload + def func5(self, x: int) -> int: ... + @overload + def func5(self, x: str) -> str: ... + def func5(self, x: Union[int, str]) -> Union[int, str]: # E: Missing return statement + """Some function.""" +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyProhibitedPropertyNonAbstract] +# flags: --strict-optional +class A: + @property + def x(self) -> int: ... # E: Missing return statement + @property + def y(self) -> int: ... # E: Missing return statement + @y.setter + def y(self, value: int) -> None: ... + +class B: + @property + def x(self) -> int: pass # E: Missing return statement + @property + def y(self) -> int: pass # E: Missing return statement + @y.setter + def y(self, value: int) -> None: pass + +class C: + @property + def x(self) -> int: # E: Missing return statement + """Some property.""" + @property + def y(self) -> int: # E: Missing return statement + """Some property.""" + @y.setter + def y(self, value: int) -> None: pass +[builtins fixtures/property.pyi] + +[case testEmptyBodyNoteABCMeta] +# flags: --strict-optional +from abc import ABC + +class A(ABC): + def foo(self) -> int: # E: Missing return statement \ + # N: If the method is meant to be abstract, use @abc.abstractmethod + ... + +[case testEmptyBodyAllowedFunctionStub] +# flags: --strict-optional +import stub +[file stub.pyi] +from typing import overload, Union + +def func1(x: str) -> int: pass +def func2(x: str) -> int: ... +def func3(x: str) -> int: + """Some function.""" + +[case testEmptyBodyAllowedMethodNonAbstractStub] +# flags: --strict-optional +import stub +[file stub.pyi] +from typing import overload, Union + +class A: + def func1(self, x: str) -> int: pass + def func2(self, x: str) -> int: ... + def func3(self, x: str) -> int: + """Some function.""" + +class B: + @classmethod + def func1(cls, x: str) -> int: pass + @classmethod + def func2(cls, x: str) -> int: ... + @classmethod + def func3(cls, x: str) -> int: + """Some function.""" +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyAllowedPropertyNonAbstractStub] +# flags: --strict-optional +import stub +[file stub.pyi] +class A: + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... + @y.setter + def y(self, value: int) -> None: ... + +class B: + @property + def x(self) -> int: pass + @property + def y(self) -> int: pass + @y.setter + def y(self, value: int) -> None: pass + +class C: + @property + def x(self) -> int: + """Some property.""" + @property + def y(self) -> int: + """Some property.""" + @y.setter + def y(self, value: int) -> None: pass +[builtins fixtures/property.pyi] + +[case testEmptyBodyAllowedMethodAbstract] +# flags: --strict-optional +from typing import overload, Union +from abc import abstractmethod + +class A: + @abstractmethod + def func1(self, x: str) -> int: pass + @abstractmethod + def func2(self, x: str) -> int: ... + @abstractmethod + def func3(self, x: str) -> int: + """Some function.""" + +class B: + @classmethod + @abstractmethod + def func1(cls, x: str) -> int: pass + @classmethod + @abstractmethod + def func2(cls, x: str) -> int: ... + @classmethod + @abstractmethod + def func3(cls, x: str) -> int: + """Some function.""" + +class C: + @overload + @abstractmethod + def func4(self, x: int) -> int: ... + @overload + @abstractmethod + def func4(self, x: str) -> str: ... + @abstractmethod + def func4(self, x: Union[int, str]) -> Union[int, str]: + pass + + @overload + @abstractmethod + def func5(self, x: int) -> int: ... + @overload + @abstractmethod + def func5(self, x: str) -> str: ... + @abstractmethod + def func5(self, x: Union[int, str]) -> Union[int, str]: + """Some function.""" +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyAllowedPropertyAbstract] +# flags: --strict-optional +from abc import abstractmethod +class A: + @property + @abstractmethod + def x(self) -> int: ... + @property + @abstractmethod + def y(self) -> int: ... + @y.setter + @abstractmethod + def y(self, value: int) -> None: ... + +class B: + @property + @abstractmethod + def x(self) -> int: pass + @property + @abstractmethod + def y(self) -> int: pass + @y.setter + @abstractmethod + def y(self, value: int) -> None: pass + +class C: + @property + @abstractmethod + def x(self) -> int: + """Some property.""" + @property + @abstractmethod + def y(self) -> int: + """Some property.""" + @y.setter + @abstractmethod + def y(self, value: int) -> None: pass +[builtins fixtures/property.pyi] + +[case testEmptyBodyImplicitlyAbstractProtocol] +# flags: --strict-optional +from typing import Protocol, overload, Union + +class P1(Protocol): + def meth(self) -> int: ... +class B1(P1): ... +class C1(P1): + def meth(self) -> int: + return 0 +B1() # E: Cannot instantiate abstract class "B1" with abstract attribute "meth" +C1() + +class P2(Protocol): + @classmethod + def meth(cls) -> int: ... +class B2(P2): ... +class C2(P2): + @classmethod + def meth(cls) -> int: + return 0 +B2() # E: Cannot instantiate abstract class "B2" with abstract attribute "meth" +C2() + +class P3(Protocol): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... +class B3(P3): ... +class C3(P3): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return 0 +B3() # E: Cannot instantiate abstract class "B3" with abstract attribute "meth" +C3() +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyImplicitlyAbstractProtocolProperty] +# flags: --strict-optional +from typing import Protocol + +class P1(Protocol): + @property + def attr(self) -> int: ... +class B1(P1): ... +class C1(P1): + @property + def attr(self) -> int: + return 0 +B1() # E: Cannot instantiate abstract class "B1" with abstract attribute "attr" +C1() + +class P2(Protocol): + @property + def attr(self) -> int: ... + @attr.setter + def attr(self, value: int) -> None: ... +class B2(P2): ... +class C2(P2): + @property + def attr(self) -> int: return 0 + @attr.setter + def attr(self, value: int) -> None: pass +B2() # E: Cannot instantiate abstract class "B2" with abstract attribute "attr" +C2() +[builtins fixtures/property.pyi] + +[case testEmptyBodyImplicitlyAbstractProtocolStub] +# flags: --strict-optional +from stub import P1, P2, P3, P4 + +class B1(P1): ... +class B2(P2): ... +class B3(P3): ... +class B4(P4): ... + +B1() +B2() +B3() +B4() # E: Cannot instantiate abstract class "B4" with abstract attribute "meth" + +[file stub.pyi] +from typing import Protocol, overload, Union +from abc import abstractmethod + +class P1(Protocol): + def meth(self) -> int: ... + +class P2(Protocol): + @classmethod + def meth(cls) -> int: ... + +class P3(Protocol): + @overload + def meth(self, x: int) -> int: ... + @overload + def meth(self, x: str) -> str: ... + +class P4(Protocol): + @abstractmethod + def meth(self) -> int: ... +[builtins fixtures/classmethod.pyi] + +[case testEmptyBodyUnsafeAbstractSuper] +# flags: --strict-optional +from stub import StubProto, StubAbstract +from typing import Protocol +from abc import abstractmethod + +class Proto(Protocol): + def meth(self) -> int: ... +class ProtoDef(Protocol): + def meth(self) -> int: return 0 + +class Abstract: + @abstractmethod + def meth(self) -> int: ... +class AbstractDef: + @abstractmethod + def meth(self) -> int: return 0 + +class SubProto(Proto): + def meth(self) -> int: + return super().meth() # E: Call to abstract method "meth" of "Proto" with trivial body via super() is unsafe +class SubProtoDef(ProtoDef): + def meth(self) -> int: + return super().meth() + +class SubAbstract(Abstract): + def meth(self) -> int: + return super().meth() # E: Call to abstract method "meth" of "Abstract" with trivial body via super() is unsafe +class SubAbstractDef(AbstractDef): + def meth(self) -> int: + return super().meth() + +class SubStubProto(StubProto): + def meth(self) -> int: + return super().meth() +class SubStubAbstract(StubAbstract): + def meth(self) -> int: + return super().meth() + +[file stub.pyi] +from typing import Protocol +from abc import abstractmethod + +class StubProto(Protocol): + def meth(self) -> int: ... +class StubAbstract: + @abstractmethod + def meth(self) -> int: ... + +[case testEmptyBodyUnsafeAbstractSuperProperty] +# flags: --strict-optional +from stub import StubProto, StubAbstract +from typing import Protocol +from abc import abstractmethod + +class Proto(Protocol): + @property + def attr(self) -> int: ... +class SubProto(Proto): + @property + def attr(self) -> int: return super().attr # E: Call to abstract method "attr" of "Proto" with trivial body via super() is unsafe + +class ProtoDef(Protocol): + @property + def attr(self) -> int: return 0 +class SubProtoDef(ProtoDef): + @property + def attr(self) -> int: return super().attr + +class Abstract: + @property + @abstractmethod + def attr(self) -> int: ... +class SubAbstract(Abstract): + @property + @abstractmethod + def attr(self) -> int: return super().attr # E: Call to abstract method "attr" of "Abstract" with trivial body via super() is unsafe + +class AbstractDef: + @property + @abstractmethod + def attr(self) -> int: return 0 +class SubAbstractDef(AbstractDef): + @property + @abstractmethod + def attr(self) -> int: return super().attr + +class SubStubProto(StubProto): + @property + def attr(self) -> int: return super().attr +class SubStubAbstract(StubAbstract): + @property + def attr(self) -> int: return super().attr + +[file stub.pyi] +from typing import Protocol +from abc import abstractmethod + +class StubProto(Protocol): + @property + def attr(self) -> int: ... +class StubAbstract: + @property + @abstractmethod + def attr(self) -> int: ... +[builtins fixtures/property.pyi] + +[case testEmptyBodyUnsafeAbstractSuperOverloads] +# flags: --strict-optional +from stub import StubProto +from typing import Protocol, overload, Union + +class ProtoEmptyImpl(Protocol): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + raise NotImplementedError +class ProtoDefImpl(Protocol): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return 0 +class ProtoNoImpl(Protocol): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + +class SubProtoEmptyImpl(ProtoEmptyImpl): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return super().meth(0) # E: Call to abstract method "meth" of "ProtoEmptyImpl" with trivial body via super() is unsafe +class SubProtoDefImpl(ProtoDefImpl): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return super().meth(0) +class SubStubProto(StubProto): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return super().meth(0) + +# TODO: it would be good to also give an error in this case. +class SubProtoNoImpl(ProtoNoImpl): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + def meth(self, x: Union[int, str]) -> Union[int, str]: + return super().meth(0) + +[file stub.pyi] +from typing import Protocol, overload + +class StubProto(Protocol): + @overload + def meth(self, x: str) -> str: ... + @overload + def meth(self, x: int) -> int: ... + +[builtins fixtures/exception.pyi] + +[case testEmptyBodyNoSuperWarningWithoutStrict] +# flags: --no-strict-optional +from typing import Protocol +from abc import abstractmethod + +class Proto(Protocol): + def meth(self) -> int: ... +class Abstract: + @abstractmethod + def meth(self) -> int: ... + +class SubProto(Proto): + def meth(self) -> int: + return super().meth() +class SubAbstract(Abstract): + def meth(self) -> int: + return super().meth() + +[case testEmptyBodyNoSuperWarningOptionalReturn] +# flags: --strict-optional +from typing import Protocol, Optional +from abc import abstractmethod + +class Proto(Protocol): + def meth(self) -> Optional[int]: pass +class Abstract: + @abstractmethod + def meth(self) -> Optional[int]: pass + +class SubProto(Proto): + def meth(self) -> Optional[int]: + return super().meth() +class SubAbstract(Abstract): + def meth(self) -> Optional[int]: + return super().meth() + +[case testEmptyBodyTypeCheckingOnly] +# flags: --strict-optional +from typing import TYPE_CHECKING + +class C: + if TYPE_CHECKING: + def dynamic(self) -> int: ... # OK diff --git a/test-data/unit/check-attr.test b/test-data/unit/check-attr.test index c5b64ee61376..fe123acfa001 100644 --- a/test-data/unit/check-attr.test +++ b/test-data/unit/check-attr.test @@ -1,4 +1,4 @@ -[case testAttrsSimple] +[case testAttrsSimple_no_empty] import attr @attr.s class A: @@ -1428,7 +1428,7 @@ class B(A): reveal_type(B) # N: Revealed type is "def (foo: builtins.int) -> __main__.B" [builtins fixtures/bool.pyi] -[case testAttrsClassHasAttributeWithAttributes] +[case testAttrsClassHasMagicAttribute] import attr @attr.s @@ -1436,14 +1436,14 @@ class A: b: int = attr.ib() c: str = attr.ib() -reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[builtins.int], attr.Attribute[builtins.str], fallback=__main__.A._AttrsAttributes]" +reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[builtins.int], attr.Attribute[builtins.str], fallback=__main__.A.____main___A_AttrsAttributes__]" reveal_type(A.__attrs_attrs__[0]) # N: Revealed type is "attr.Attribute[builtins.int]" reveal_type(A.__attrs_attrs__.b) # N: Revealed type is "attr.Attribute[builtins.int]" -A.__attrs_attrs__.x # E: "_AttrsAttributes" has no attribute "x" +A.__attrs_attrs__.x # E: "____main___A_AttrsAttributes__" has no attribute "x" [builtins fixtures/attr.pyi] -[case testAttrsBareClassHasAttributeWithAttributes] +[case testAttrsBareClassHasMagicAttribute] import attr @attr.s @@ -1451,14 +1451,14 @@ class A: b = attr.ib() c = attr.ib() -reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[Any], attr.Attribute[Any], fallback=__main__.A._AttrsAttributes]" +reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[Any], attr.Attribute[Any], fallback=__main__.A.____main___A_AttrsAttributes__]" reveal_type(A.__attrs_attrs__[0]) # N: Revealed type is "attr.Attribute[Any]" reveal_type(A.__attrs_attrs__.b) # N: Revealed type is "attr.Attribute[Any]" -A.__attrs_attrs__.x # E: "_AttrsAttributes" has no attribute "x" +A.__attrs_attrs__.x # E: "____main___A_AttrsAttributes__" has no attribute "x" [builtins fixtures/attr.pyi] -[case testAttrsNGClassHasAttributeWithAttributes] +[case testAttrsNGClassHasMagicAttribute] import attr @attr.define @@ -1466,10 +1466,53 @@ class A: b: int c: str -reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[builtins.int], attr.Attribute[builtins.str], fallback=__main__.A._AttrsAttributes]" +reveal_type(A.__attrs_attrs__) # N: Revealed type is "Tuple[attr.Attribute[builtins.int], attr.Attribute[builtins.str], fallback=__main__.A.____main___A_AttrsAttributes__]" reveal_type(A.__attrs_attrs__[0]) # N: Revealed type is "attr.Attribute[builtins.int]" reveal_type(A.__attrs_attrs__.b) # N: Revealed type is "attr.Attribute[builtins.int]" -A.__attrs_attrs__.x # E: "_AttrsAttributes" has no attribute "x" +A.__attrs_attrs__.x # E: "____main___A_AttrsAttributes__" has no attribute "x" + +[builtins fixtures/attr.pyi] + +[case testAttrsMagicAttributeProtocol] +import attr +from typing import Any, Protocol, Type, ClassVar + +class AttrsInstance(Protocol): + __attrs_attrs__: ClassVar[Any] + +@attr.define +class A: + b: int + c: str + +def takes_attrs_cls(cls: Type[AttrsInstance]) -> None: + pass + +def takes_attrs_instance(inst: AttrsInstance) -> None: + pass + +takes_attrs_cls(A) +takes_attrs_instance(A(1, "")) + +takes_attrs_cls(A(1, "")) # E: Argument 1 to "takes_attrs_cls" has incompatible type "A"; expected "Type[AttrsInstance]" +takes_attrs_instance(A) # E: Argument 1 to "takes_attrs_instance" has incompatible type "Type[A]"; expected "AttrsInstance" # N: ClassVar protocol member AttrsInstance.__attrs_attrs__ can never be matched by a class object +[builtins fixtures/attr.pyi] + +[case testAttrsInitMethodAlwaysGenerates] +from typing import Tuple +import attr + +@attr.define(init=False) +class A: + b: int + c: str + def __init__(self, bc: Tuple[int, str]) -> None: + b, c = bc + self.__attrs_init__(b, c) + +reveal_type(A) # N: Revealed type is "def (bc: Tuple[builtins.int, builtins.str]) -> __main__.A" +reveal_type(A.__init__) # N: Revealed type is "def (self: __main__.A, bc: Tuple[builtins.int, builtins.str])" +reveal_type(A.__attrs_init__) # N: Revealed type is "def (self: __main__.A, b: builtins.int, c: builtins.str)" [builtins fixtures/attr.pyi] diff --git a/test-data/unit/check-basic.test b/test-data/unit/check-basic.test index 238aab3944ff..a4056c8cb576 100644 --- a/test-data/unit/check-basic.test +++ b/test-data/unit/check-basic.test @@ -300,12 +300,15 @@ main:4: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "i main:5: error: Incompatible return value type (got "int", expected "str") main:6: error: Argument 1 to "f" of "A" has incompatible type "str"; expected "int" -[case testTrailingCommaParsing-skip] +[case testTrailingCommaParsing] x = 1 -x in 1, -if x in 1, : - pass +x in 1, # E: Unsupported right operand type for in ("int") +[builtins fixtures/tuple.pyi] + +[case testTrailingCommaInIfParsing] +if x in 1, : pass [out] +main:1: error: invalid syntax [case testInitReturnTypeError] class C: @@ -347,7 +350,8 @@ from typing import Union class A: ... class B: ... -x: Union[mock, A] # E: Module "mock" is not valid as a type +x: Union[mock, A] # E: Module "mock" is not valid as a type \ + # N: Perhaps you meant to use a protocol matching the module structure? if isinstance(x, B): pass @@ -363,7 +367,8 @@ from typing import overload, Any, Union @overload def f(x: int) -> int: ... @overload -def f(x: str) -> Union[mock, str]: ... # E: Module "mock" is not valid as a type +def f(x: str) -> Union[mock, str]: ... # E: Module "mock" is not valid as a type \ + # N: Perhaps you meant to use a protocol matching the module structure? def f(x): pass @@ -388,15 +393,6 @@ b = none.__bool__() reveal_type(b) # N: Revealed type is "Literal[False]" [builtins fixtures/bool.pyi] -[case testNoneHasBoolShowNoneErrorsFalse] -none = None -b = none.__bool__() -reveal_type(b) # N: Revealed type is "Literal[False]" -[builtins fixtures/bool.pyi] -[file mypy.ini] -\[mypy] -show_none_errors = False - [case testAssignmentInvariantNoteForList] from typing import List x: List[int] diff --git a/test-data/unit/check-bound.test b/test-data/unit/check-bound.test index bf13ef874579..eb97bde32e1f 100644 --- a/test-data/unit/check-bound.test +++ b/test-data/unit/check-bound.test @@ -215,3 +215,13 @@ if int(): b = 'a' # E: Incompatible types in assignment (expression has type "str", variable has type "int") twice(a) # E: Value of type variable "T" of "twice" cannot be "int" [builtins fixtures/args.pyi] + + +[case testIterableBoundUnpacking] +from typing import Tuple, TypeVar +TupleT = TypeVar("TupleT", bound=Tuple[int, ...]) +def f(t: TupleT) -> None: + a, *b = t + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(b) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test index ecc81f3cee33..8e0545953bd8 100644 --- a/test-data/unit/check-class-namedtuple.test +++ b/test-data/unit/check-class-namedtuple.test @@ -585,8 +585,8 @@ class Base(NamedTuple): reveal_type(self[T]) # N: Revealed type is "builtins.int" \ # E: No overload variant of "__getitem__" of "tuple" matches argument type "object" \ # N: Possible overload variants: \ - # N: def __getitem__(self, int) -> int \ - # N: def __getitem__(self, slice) -> Tuple[int, ...] + # N: def __getitem__(self, int, /) -> int \ + # N: def __getitem__(self, slice, /) -> Tuple[int, ...] return self.x def bad_override(self) -> int: return self.x diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 8adf2e7ed5f1..b16387f194d4 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -865,6 +865,23 @@ class C(B): pass class D(C): pass class D2(C): pass +[case testConstructorJoinsWithCustomMetaclass] +# flags: --strict-optional +from typing import TypeVar +import abc + +def func() -> None: pass +class NormalClass: pass +class WithMetaclass(metaclass=abc.ABCMeta): pass + +T = TypeVar('T') +def join(x: T, y: T) -> T: pass + +f1 = join(func, WithMetaclass) +reveal_type(f1()) # N: Revealed type is "Union[__main__.WithMetaclass, None]" + +f2 = join(WithMetaclass, func) +reveal_type(f2()) # N: Revealed type is "Union[__main__.WithMetaclass, None]" -- Attribute access in class body -- ------------------------------ @@ -1060,6 +1077,35 @@ def f() -> None: a.g(a) # E: Too many arguments for "g" of "A" [targets __main__, __main__.f] +[case testGenericClassWithinFunction] +from typing import TypeVar + +def test() -> None: + T = TypeVar('T', bound='Foo') + class Foo: + def returns_int(self) -> int: + return 0 + + def bar(self, foo: T) -> T: + x: T = foo + reveal_type(x) # N: Revealed type is "T`-1" + reveal_type(x.returns_int()) # N: Revealed type is "builtins.int" + return foo + reveal_type(Foo.bar) # N: Revealed type is "def [T <: __main__.Foo@5] (self: __main__.Foo@5, foo: T`-1) -> T`-1" + +[case testGenericClassWithInvalidTypevarUseWithinFunction] +from typing import TypeVar + +def test() -> None: + T = TypeVar('T', bound='Foo') + class Foo: + invalid: T # E: Type variable "T" is unbound \ + # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \ + # N: (Hint: Use "T" in function signature to bind "T" inside a function) + + def bar(self, foo: T) -> T: + pass + [case testConstructNestedClass] import typing class A: @@ -1143,7 +1189,7 @@ reveal_type(Foo().Meta.name) # N: Revealed type is "builtins.str" class A: def __init__(self): - self.x = None # type: int + self.x = None # type: int # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs a = None # type: A a.x = 1 a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") @@ -1155,7 +1201,7 @@ a.x = 1 a.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") class A: def __init__(self): - self.x = None # type: int + self.x = None # type: int # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs -- Special cases @@ -1897,12 +1943,12 @@ class B(A): [out] tmp/foo.pyi:5: error: Signature of "__add__" incompatible with supertype "A" tmp/foo.pyi:5: note: Superclass: -tmp/foo.pyi:5: note: def __add__(self, int) -> int +tmp/foo.pyi:5: note: def __add__(self, int, /) -> int tmp/foo.pyi:5: note: Subclass: tmp/foo.pyi:5: note: @overload -tmp/foo.pyi:5: note: def __add__(self, int) -> int +tmp/foo.pyi:5: note: def __add__(self, int, /) -> int tmp/foo.pyi:5: note: @overload -tmp/foo.pyi:5: note: def __add__(self, str) -> str +tmp/foo.pyi:5: note: def __add__(self, str, /) -> str tmp/foo.pyi:5: note: Overloaded operator methods cannot have wider argument types in overrides [case testOperatorMethodOverrideWideningArgumentType] @@ -2012,16 +2058,16 @@ class B(A): tmp/foo.pyi:8: error: Signature of "__add__" incompatible with supertype "A" tmp/foo.pyi:8: note: Superclass: tmp/foo.pyi:8: note: @overload -tmp/foo.pyi:8: note: def __add__(self, int) -> A +tmp/foo.pyi:8: note: def __add__(self, int, /) -> A tmp/foo.pyi:8: note: @overload -tmp/foo.pyi:8: note: def __add__(self, str) -> A +tmp/foo.pyi:8: note: def __add__(self, str, /) -> A tmp/foo.pyi:8: note: Subclass: tmp/foo.pyi:8: note: @overload -tmp/foo.pyi:8: note: def __add__(self, int) -> A +tmp/foo.pyi:8: note: def __add__(self, int, /) -> A tmp/foo.pyi:8: note: @overload -tmp/foo.pyi:8: note: def __add__(self, str) -> A +tmp/foo.pyi:8: note: def __add__(self, str, /) -> A tmp/foo.pyi:8: note: @overload -tmp/foo.pyi:8: note: def __add__(self, type) -> A +tmp/foo.pyi:8: note: def __add__(self, type, /) -> A tmp/foo.pyi:8: note: Overloaded operator methods cannot have wider argument types in overrides [case testOverloadedOperatorMethodOverrideWithSwitchedItemOrder] @@ -2348,9 +2394,9 @@ a: Union[int, float] b: int c: float -reveal_type(a + a) # N: Revealed type is "builtins.float" -reveal_type(a + b) # N: Revealed type is "builtins.float" -reveal_type(b + a) # N: Revealed type is "builtins.float" +reveal_type(a + a) # N: Revealed type is "Union[builtins.int, builtins.float]" +reveal_type(a + b) # N: Revealed type is "Union[builtins.int, builtins.float]" +reveal_type(b + a) # N: Revealed type is "Union[builtins.int, builtins.float]" reveal_type(a + c) # N: Revealed type is "builtins.float" reveal_type(c + a) # N: Revealed type is "builtins.float" [builtins fixtures/ops.pyi] @@ -2489,8 +2535,8 @@ def sum(x: Iterable[T]) -> Union[T, int]: ... def len(x: Iterable[T]) -> int: ... x = [1.1, 2.2, 3.3] -reveal_type(sum(x)) # N: Revealed type is "builtins.float" -reveal_type(sum(x) / len(x)) # N: Revealed type is "builtins.float" +reveal_type(sum(x)) # N: Revealed type is "Union[builtins.float, builtins.int]" +reveal_type(sum(x) / len(x)) # N: Revealed type is "Union[builtins.float, builtins.int]" [builtins fixtures/floatdict.pyi] [case testOperatorWithEmptyListAndSum] @@ -2933,8 +2979,9 @@ class B: pass [case testConstructInstanceWith__new__] +from typing import Optional class C: - def __new__(cls, foo: int = None) -> 'C': + def __new__(cls, foo: Optional[int] = None) -> 'C': obj = object.__new__(cls) return obj @@ -3232,7 +3279,8 @@ def error(u_c: Type[U]) -> P: # Error here, see below return new_pro(u_c) # Error here, see below [out] main:11: note: Revealed type is "__main__.WizUser" -main:12: error: A function returning TypeVar should receive at least one argument containing the same Typevar +main:12: error: A function returning TypeVar should receive at least one argument containing the same TypeVar +main:12: note: Consider using the upper bound "ProUser" instead main:13: error: Value of type variable "P" of "new_pro" cannot be "U" main:13: error: Incompatible return value type (got "U", expected "P") @@ -4317,7 +4365,7 @@ class C(B): x = object() [out] main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") -main:6: error: Incompatible types in assignment (expression has type "object", base class "B" defined the type as "str") +main:6: error: Incompatible types in assignment (expression has type "object", base class "A" defined the type as "int") [case testClassOneErrorPerLine] class A: @@ -4327,7 +4375,7 @@ class B(A): x = 1.0 [out] main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") -main:5: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") +main:5: error: Incompatible types in assignment (expression has type "float", base class "A" defined the type as "int") [case testClassIgnoreType_RedefinedAttributeAndGrandparentAttributeTypesNotIgnored] class A: @@ -4335,7 +4383,7 @@ class A: class B(A): x = '' # type: ignore class C(B): - x = '' + x = '' # E: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") [out] [case testClassIgnoreType_RedefinedAttributeTypeIgnoredInChildren] @@ -4351,7 +4399,7 @@ class C(B): class X(type): pass class Y(type): pass class A(metaclass=X): pass -class B(A, metaclass=Y): pass # E: Inconsistent metaclass structure for "B" +class B(A, metaclass=Y): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [case testMetaclassNoTypeReveal] class M: @@ -4392,6 +4440,35 @@ def f(TB: Type[B]): reveal_type(TB) # N: Revealed type is "Type[__main__.B]" reveal_type(TB.x) # N: Revealed type is "builtins.int" +[case testMetaclassAsAny] +from typing import Any, ClassVar + +MyAny: Any +class WithMeta(metaclass=MyAny): + x: ClassVar[int] + +reveal_type(WithMeta.a) # N: Revealed type is "Any" +reveal_type(WithMeta.m) # N: Revealed type is "Any" +reveal_type(WithMeta.x) # N: Revealed type is "builtins.int" +reveal_type(WithMeta().x) # N: Revealed type is "builtins.int" +WithMeta().m # E: "WithMeta" has no attribute "m" +WithMeta().a # E: "WithMeta" has no attribute "a" + +[case testMetaclassAsAnyWithAFlag] +# flags: --disallow-subclassing-any +from typing import Any, ClassVar + +MyAny: Any +class WithMeta(metaclass=MyAny): # E: Class cannot use "__main__.MyAny" as a metaclass (has type "Any") + x: ClassVar[int] + +reveal_type(WithMeta.a) # N: Revealed type is "Any" +reveal_type(WithMeta.m) # N: Revealed type is "Any" +reveal_type(WithMeta.x) # N: Revealed type is "builtins.int" +reveal_type(WithMeta().x) # N: Revealed type is "builtins.int" +WithMeta().m # E: "WithMeta" has no attribute "m" +WithMeta().a # E: "WithMeta" has no attribute "a" + [case testMetaclassIterable] from typing import Iterable, Iterator @@ -4476,15 +4553,7 @@ from missing import M class A(metaclass=M): y = 0 reveal_type(A.y) # N: Revealed type is "builtins.int" -A.x # E: "Type[A]" has no attribute "x" - -[case testAnyMetaclass] -from typing import Any -M = None # type: Any -class A(metaclass=M): - y = 0 -reveal_type(A.y) # N: Revealed type is "builtins.int" -A.x # E: "Type[A]" has no attribute "x" +reveal_type(A.x) # N: Revealed type is "Any" [case testValidTypeAliasAsMetaclass] from typing_extensions import TypeAlias @@ -4710,7 +4779,7 @@ class A(Tuple[int, str]): pass -- ----------------------- [case testCrashOnSelfRecursiveNamedTupleVar] - +# flags: --disable-recursive-aliases from typing import NamedTuple N = NamedTuple('N', [('x', N)]) # E: Cannot resolve name "N" (possible cyclic definition) @@ -4740,7 +4809,7 @@ lst = [n, m] [builtins fixtures/isinstancelist.pyi] [case testCorrectJoinOfSelfRecursiveTypedDicts] - +# flags: --disable-recursive-aliases from mypy_extensions import TypedDict class N(TypedDict): @@ -5213,8 +5282,8 @@ class CD(six.with_metaclass(M)): pass # E: Multiple metaclass definitions class M1(type): pass class Q1(metaclass=M1): pass @six.add_metaclass(M) -class CQA(Q1): pass # E: Inconsistent metaclass structure for "CQA" -class CQW(six.with_metaclass(M, Q1)): pass # E: Inconsistent metaclass structure for "CQW" +class CQA(Q1): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class CQW(six.with_metaclass(M, Q1)): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [builtins fixtures/tuple.pyi] [case testSixMetaclassAny] @@ -5319,7 +5388,7 @@ class C5(future.utils.with_metaclass(f())): pass # E: Dynamic metaclass not sup class M1(type): pass class Q1(metaclass=M1): pass -class CQW(future.utils.with_metaclass(M, Q1)): pass # E: Inconsistent metaclass structure for "CQW" +class CQW(future.utils.with_metaclass(M, Q1)): pass # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [builtins fixtures/tuple.pyi] [case testFutureMetaclassAny] @@ -5497,6 +5566,13 @@ class E(Protocol): # OK, is a protocol class F(E, Protocol): # OK, is a protocol pass +# Custom metaclass subclassing `ABCMeta`, see #13561 +class CustomMeta(ABCMeta): + pass + +class G(A, metaclass=CustomMeta): # Ok, has CustomMeta as a metaclass + pass + [file b.py] # All of these are OK because this is not a stub file. from abc import ABCMeta, abstractmethod @@ -5525,6 +5601,12 @@ class E(Protocol): class F(E, Protocol): pass +class CustomMeta(ABCMeta): + pass + +class G(A, metaclass=CustomMeta): + pass + [case testClassMethodOverride] from typing import Callable, Any @@ -6663,6 +6745,81 @@ class MyMetaClass(type): class MyClass(metaclass=MyMetaClass): pass + +[case testMetaclassPlaceholderNode] +from sympy.assumptions import ManagedProperties +from sympy.ops import AssocOp +reveal_type(AssocOp.x) # N: Revealed type is "sympy.basic.Basic" +reveal_type(AssocOp.y) # N: Revealed type is "builtins.int" + +[file sympy/__init__.py] + +[file sympy/assumptions.py] +from .basic import Basic +class ManagedProperties(type): + x: Basic + y: int +# The problem is with the next line, +# it creates the following order (classname, metaclass): +# 1. Basic NameExpr(ManagedProperties) +# 2. AssocOp None +# 3. ManagedProperties None +# 4. Basic NameExpr(ManagedProperties [sympy.assumptions.ManagedProperties]) +# So, `AssocOp` will still have `metaclass_type` as `None` +# and all its `mro` types will have `declared_metaclass` as `None`. +from sympy.ops import AssocOp + +[file sympy/basic.py] +from .assumptions import ManagedProperties +class Basic(metaclass=ManagedProperties): ... + +[file sympy/ops.py] +from sympy.basic import Basic +class AssocOp(Basic): ... + +[case testMetaclassSubclassSelf] +# This does not make much sense, but we must not crash: +import a +[file m.py] +from a import A # E: Module "a" has no attribute "A" +class Meta(A): pass +[file a.py] +from m import Meta +class A(metaclass=Meta): pass + +[case testMetaclassConflict] +class MyMeta1(type): ... +class MyMeta2(type): ... +class MyMeta3(type): ... +class A(metaclass=MyMeta1): ... +class B(metaclass=MyMeta2): ... +class C(metaclass=type): ... +class A1(A): ... +class E: ... + +class CorrectMeta(MyMeta1, MyMeta2): ... +class CorrectSubclass1(A1, B, E, metaclass=CorrectMeta): ... +class CorrectSubclass2(A, B, E, metaclass=CorrectMeta): ... +class CorrectSubclass3(B, A, metaclass=CorrectMeta): ... + +class ChildOfCorrectSubclass1(CorrectSubclass1): ... + +class CorrectWithType1(C, A1): ... +class CorrectWithType2(B, C): ... + +class Conflict1(A1, B, E): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class Conflict2(A, B): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class Conflict3(B, A): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + +class ChildOfConflict1(Conflict3): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases +class ChildOfConflict2(Conflict3, metaclass=CorrectMeta): ... + +class ConflictingMeta(MyMeta1, MyMeta3): ... +class Conflict4(A1, B, E, metaclass=ConflictingMeta): ... # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + +class ChildOfCorrectButWrongMeta(CorrectSubclass1, metaclass=ConflictingMeta): # E: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases + ... + [case testGenericOverride] from typing import Generic, TypeVar, Any @@ -7296,6 +7453,74 @@ def meth1(self: Any, y: str) -> str: ... T = TypeVar("T") def meth2(self: Any, y: T) -> T: ... +[case testNewAndInitNoReturn] +from typing import NoReturn + +class A: + def __new__(cls) -> NoReturn: ... + +class B: + def __init__(self) -> NoReturn: ... + +class C: + def __new__(cls) -> "C": ... + def __init__(self) -> NoReturn: ... + +class D: + def __new__(cls) -> NoReturn: ... + def __init__(self) -> NoReturn: ... + +reveal_type(A()) # N: Revealed type is "" +reveal_type(B()) # N: Revealed type is "" +reveal_type(C()) # N: Revealed type is "" +reveal_type(D()) # N: Revealed type is "" + +[case testOverloadedNewAndInitNoReturn] +from typing import NoReturn, overload + +class A: + @overload + def __new__(cls) -> NoReturn: ... + @overload + def __new__(cls, a: int) -> "A": ... + def __new__(cls, a: int = ...) -> "A": ... + +class B: + @overload + def __init__(self) -> NoReturn: ... + @overload + def __init__(self, a: int) -> None: ... + def __init__(self, a: int = ...) -> None: ... + +class C: + def __new__(cls, a: int = ...) -> "C": ... + @overload + def __init__(self) -> NoReturn: ... + @overload + def __init__(self, a: int) -> None: ... + def __init__(self, a: int = ...) -> None: ... + +class D: + @overload + def __new__(cls) -> NoReturn: ... + @overload + def __new__(cls, a: int) -> "D": ... + def __new__(cls, a: int = ...) -> "D": ... + @overload + def __init__(self) -> NoReturn: ... + @overload + def __init__(self, a: int) -> None: ... + def __init__(self, a: int = ...) -> None: ... + +reveal_type(A()) # N: Revealed type is "" +reveal_type(A(1)) # N: Revealed type is "__main__.A" +reveal_type(B()) # N: Revealed type is "" +reveal_type(B(1)) # N: Revealed type is "__main__.B" +reveal_type(C()) # N: Revealed type is "" +reveal_type(C(1)) # N: Revealed type is "__main__.C" +reveal_type(D()) # N: Revealed type is "" +reveal_type(D(1)) # N: Revealed type is "__main__.D" + [case testClassScopeImportWithWrapperAndError] class Foo: from mod import foo # E: Unsupported class scoped import @@ -7366,3 +7591,46 @@ class D(C[List[T]]): ... di: D[int] reveal_type(di) # N: Revealed type is "Tuple[builtins.list[builtins.int], builtins.list[builtins.int], fallback=__main__.D[builtins.int]]" [builtins fixtures/tuple.pyi] + +[case testOverrideAttrWithSettableProperty] +class Foo: + def __init__(self) -> None: + self.x = 42 + +class Bar(Foo): + @property + def x(self) -> int: ... + @x.setter + def x(self, value: int) -> None: ... +[builtins fixtures/property.pyi] + +[case testOverrideAttrWithSettablePropertyAnnotation] +class Foo: + x: int + +class Bar(Foo): + @property + def x(self) -> int: ... + @x.setter + def x(self, value: int) -> None: ... +[builtins fixtures/property.pyi] + +[case testOverrideMethodProperty] +class B: + def foo(self) -> int: + ... +class C(B): + @property + def foo(self) -> int: # E: Signature of "foo" incompatible with supertype "B" + ... +[builtins fixtures/property.pyi] + +[case testOverridePropertyMethod] +class B: + @property + def foo(self) -> int: + ... +class C(B): + def foo(self) -> int: # E: Signature of "foo" incompatible with supertype "B" + ... +[builtins fixtures/property.pyi] diff --git a/test-data/unit/check-ctypes.test b/test-data/unit/check-ctypes.test index 605c54fb5694..5a350256f8e9 100644 --- a/test-data/unit/check-ctypes.test +++ b/test-data/unit/check-ctypes.test @@ -15,8 +15,8 @@ a[1] = ctypes.c_int(42) a[2] = MyCInt(42) a[3] = b"bytes" # E: No overload variant of "__setitem__" of "Array" matches argument types "int", "bytes" \ # N: Possible overload variants: \ - # N: def __setitem__(self, int, Union[c_int, int]) -> None \ - # N: def __setitem__(self, slice, List[Union[c_int, int]]) -> None + # N: def __setitem__(self, int, Union[c_int, int], /) -> None \ + # N: def __setitem__(self, slice, List[Union[c_int, int]], /) -> None for x in a: reveal_type(x) # N: Revealed type is "builtins.int" [builtins fixtures/floatdict.pyi] @@ -38,13 +38,13 @@ reveal_type(mya[1:3]) # N: Revealed type is "builtins.list[__main__.MyCInt]" mya[0] = 42 mya[1] = ctypes.c_int(42) # E: No overload variant of "__setitem__" of "Array" matches argument types "int", "c_int" \ # N: Possible overload variants: \ - # N: def __setitem__(self, int, Union[MyCInt, int]) -> None \ - # N: def __setitem__(self, slice, List[Union[MyCInt, int]]) -> None + # N: def __setitem__(self, int, Union[MyCInt, int], /) -> None \ + # N: def __setitem__(self, slice, List[Union[MyCInt, int]], /) -> None mya[2] = MyCInt(42) mya[3] = b"bytes" # E: No overload variant of "__setitem__" of "Array" matches argument types "int", "bytes" \ # N: Possible overload variants: \ - # N: def __setitem__(self, int, Union[MyCInt, int]) -> None \ - # N: def __setitem__(self, slice, List[Union[MyCInt, int]]) -> None + # N: def __setitem__(self, int, Union[MyCInt, int], /) -> None \ + # N: def __setitem__(self, slice, List[Union[MyCInt, int]], /) -> None for myx in mya: reveal_type(myx) # N: Revealed type is "__main__.MyCInt" @@ -71,8 +71,8 @@ mya[1] = ctypes.c_uint(42) mya[2] = MyCInt(42) mya[3] = b"bytes" # E: No overload variant of "__setitem__" of "Array" matches argument types "int", "bytes" \ # N: Possible overload variants: \ - # N: def __setitem__(self, int, Union[MyCInt, int, c_uint]) -> None \ - # N: def __setitem__(self, slice, List[Union[MyCInt, int, c_uint]]) -> None + # N: def __setitem__(self, int, Union[MyCInt, int, c_uint], /) -> None \ + # N: def __setitem__(self, slice, List[Union[MyCInt, int, c_uint]], /) -> None for myx in mya: reveal_type(myx) # N: Revealed type is "Union[__main__.MyCInt, builtins.int]" [builtins fixtures/floatdict.pyi] diff --git a/test-data/unit/check-custom-plugin.test b/test-data/unit/check-custom-plugin.test index ee19113f000f..a716109d345e 100644 --- a/test-data/unit/check-custom-plugin.test +++ b/test-data/unit/check-custom-plugin.test @@ -991,3 +991,17 @@ class Cls: [file mypy.ini] \[mypy] plugins=/test-data/unit/plugins/class_attr_hook.py + +[case testAddClassMethodPlugin] +# flags: --config-file tmp/mypy.ini +class BaseAddMethod: pass + +class MyClass(BaseAddMethod): + pass + +my_class = MyClass() +reveal_type(MyClass.foo_classmethod) # N: Revealed type is "def ()" +reveal_type(MyClass.foo_staticmethod) # N: Revealed type is "def (builtins.int) -> builtins.str" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/add_classmethod.py diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 6abb5597e464..3ec4c60e6929 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -187,6 +187,66 @@ reveal_type(C) # N: Revealed type is "def (some_int: builtins.int, some_str: bu [builtins fixtures/dataclasses.pyi] +[case testDataclassIncompatibleOverrides] +# flags: --python-version 3.7 +from dataclasses import dataclass + +@dataclass +class Base: + foo: int + +@dataclass +class BadDerived1(Base): + def foo(self) -> int: # E: Dataclass attribute may only be overridden by another attribute \ + # E: Signature of "foo" incompatible with supertype "Base" + return 1 + +@dataclass +class BadDerived2(Base): + @property # E: Dataclass attribute may only be overridden by another attribute + def foo(self) -> int: # E: Cannot override writeable attribute with read-only property + return 2 + +@dataclass +class BadDerived3(Base): + class foo: pass # E: Dataclass attribute may only be overridden by another attribute +[builtins fixtures/dataclasses.pyi] + +[case testDataclassMultipleInheritance] +# flags: --python-version 3.7 +from dataclasses import dataclass + +class Unrelated: + foo: str + +@dataclass +class Base: + bar: int + +@dataclass +class Derived(Base, Unrelated): + pass + +d = Derived(3) +reveal_type(d.foo) # N: Revealed type is "builtins.str" +reveal_type(d.bar) # N: Revealed type is "builtins.int" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassIncompatibleFrozenOverride] +# flags: --python-version 3.7 +from dataclasses import dataclass + +@dataclass(frozen=True) +class Base: + foo: int + +@dataclass(frozen=True) +class BadDerived(Base): + @property # E: Dataclass attribute may only be overridden by another attribute + def foo(self) -> int: + return 3 +[builtins fixtures/dataclasses.pyi] + [case testDataclassesFreezing] # flags: --python-version 3.7 from dataclasses import dataclass @@ -200,6 +260,28 @@ john.name = 'Ben' # E: Property "name" defined in "Person" is read-only [builtins fixtures/dataclasses.pyi] +[case testDataclassesInconsistentFreezing] +# flags: --python-version 3.7 +from dataclasses import dataclass + +@dataclass(frozen=True) +class FrozenBase: + pass + +@dataclass +class BadNormalDerived(FrozenBase): # E: Cannot inherit non-frozen dataclass from a frozen one + pass + +@dataclass +class NormalBase: + pass + +@dataclass(frozen=True) +class BadFrozenDerived(NormalBase): # E: Cannot inherit frozen dataclass from a non-frozen one + pass + +[builtins fixtures/dataclasses.pyi] + [case testDataclassesFields] # flags: --python-version 3.7 from dataclasses import dataclass, field @@ -550,8 +632,8 @@ class Two: S: TypeAlias = Callable[[int], str] # E: Type aliases inside dataclass definitions are not supported at runtime c = Two() -x = c.S # E: Member "S" is not assignable -reveal_type(x) # N: Revealed type is "Any" +x = c.S +reveal_type(x) # N: Revealed type is "builtins.object" [builtins fixtures/dataclasses.pyi] [case testDataclassOrdering] @@ -1283,10 +1365,10 @@ from dataclasses import dataclass class A: foo: int -@dataclass +@dataclass(frozen=True) class B(A): - @property - def foo(self) -> int: pass # E: Signature of "foo" incompatible with supertype "A" + @property # E: Dataclass attribute may only be overridden by another attribute + def foo(self) -> int: pass reveal_type(B) # N: Revealed type is "def (foo: builtins.int) -> __main__.B" @@ -1559,7 +1641,7 @@ A(a=func).a = func # E: Property "a" defined in "A" is read-only # flags: --python-version 3.7 from dataclasses import dataclass -def foo(): +def foo() -> None: @dataclass class Foo: foo: int @@ -1737,10 +1819,10 @@ c.x # E: "C" has no attribute "x" [case testDataclassCheckTypeVarBounds] # flags: --python-version 3.7 from dataclasses import dataclass -from typing import Protocol, Dict, TypeVar, Generic +from typing import ClassVar, Protocol, Dict, TypeVar, Generic class DataclassProtocol(Protocol): - __dataclass_fields__: Dict + __dataclass_fields__: ClassVar[Dict] T = TypeVar("T", bound=DataclassProtocol) @@ -1796,3 +1878,57 @@ t: Two reveal_type(t.__match_args__) # E: "Two" has no attribute "__match_args__" \ # N: Revealed type is "Any" [builtins fixtures/dataclasses.pyi] + +[case testFinalInDataclass] +from dataclasses import dataclass +from typing import Final + +@dataclass +class FirstClass: + FIRST_CONST: Final = 3 # OK + +@dataclass +class SecondClass: + SECOND_CONST: Final = FirstClass.FIRST_CONST # E: Need type argument for Final[...] with non-literal default in dataclass + +reveal_type(FirstClass().FIRST_CONST) # N: Revealed type is "Literal[3]?" +FirstClass().FIRST_CONST = 42 # E: Cannot assign to final attribute "FIRST_CONST" +reveal_type(SecondClass().SECOND_CONST) # N: Revealed type is "Literal[3]?" +SecondClass().SECOND_CONST = 42 # E: Cannot assign to final attribute "SECOND_CONST" +[builtins fixtures/dataclasses.pyi] + +[case testDataclassFieldsProtocol] +# flags: --python-version 3.9 +from dataclasses import dataclass +from typing import Any, Protocol + +class ConfigProtocol(Protocol): + __dataclass_fields__: dict[str, Any] + +def takes_cp(cp: ConfigProtocol): ... + +@dataclass +class MyDataclass: + x: int = 3 + +takes_cp(MyDataclass) +[builtins fixtures/dataclasses.pyi] + +[case testDataclassTypeAnnotationAliasUpdated] +import a +[file a.py] +from dataclasses import dataclass +from b import B + +@dataclass +class D: + x: B + +reveal_type(D) # N: Revealed type is "def (x: builtins.list[b.C]) -> a.D" +[file b.py] +from typing import List +import a +B = List[C] +class C(CC): ... +class CC: ... +[builtins fixtures/dataclasses.pyi] diff --git a/test-data/unit/check-default-plugin.test b/test-data/unit/check-default-plugin.test deleted file mode 100644 index 4d8844d254d1..000000000000 --- a/test-data/unit/check-default-plugin.test +++ /dev/null @@ -1,84 +0,0 @@ --- Test cases for the default plugin --- --- Note that we have additional test cases in pythoneval.test (that use real typeshed stubs). - - -[case testContextManagerWithGenericFunction] -from contextlib import contextmanager -from typing import TypeVar, Iterator - -T = TypeVar('T') - -@contextmanager -def yield_id(item: T) -> Iterator[T]: - yield item - -reveal_type(yield_id) # N: Revealed type is "def [T] (item: T`-1) -> contextlib.GeneratorContextManager[T`-1]" - -with yield_id(1) as x: - reveal_type(x) # N: Revealed type is "builtins.int" - -f = yield_id -def g(x, y): pass -f = g # E: Incompatible types in assignment (expression has type "Callable[[Any, Any], Any]", variable has type "Callable[[T], GeneratorContextManager[T]]") -[typing fixtures/typing-medium.pyi] -[builtins fixtures/tuple.pyi] - -[case testAsyncContextManagerWithGenericFunction] -# flags: --python-version 3.7 -from contextlib import asynccontextmanager -from typing import TypeVar, AsyncGenerator - -T = TypeVar('T') - -@asynccontextmanager -async def yield_id(item: T) -> AsyncGenerator[T, None]: - yield item - -reveal_type(yield_id) # N: Revealed type is "def [T] (item: T`-1) -> typing.AsyncContextManager[T`-1]" - -async def f() -> None: - async with yield_id(1) as x: - reveal_type(x) # N: Revealed type is "builtins.int" -[typing fixtures/typing-async.pyi] -[builtins fixtures/tuple.pyi] - -[case testContextManagerReturnsGenericFunction] -import contextlib -from typing import Callable -from typing import Generator -from typing import Iterable -from typing import TypeVar - -TArg = TypeVar('TArg') -TRet = TypeVar('TRet') - -@contextlib.contextmanager -def _thread_mapper(maxsize: int) -> Generator[ - Callable[[Callable[[TArg], TRet], Iterable[TArg]], Iterable[TRet]], - None, None, -]: - # defined inline as there isn't a builtins.map fixture - def my_map(f: Callable[[TArg], TRet], it: Iterable[TArg]) -> Iterable[TRet]: - for x in it: - yield f(x) - - yield my_map - -def identity(x: int) -> int: return x - -with _thread_mapper(1) as m: - lst = list(m(identity, [2, 3])) - reveal_type(lst) # N: Revealed type is "builtins.list[builtins.int]" -[typing fixtures/typing-medium.pyi] -[builtins fixtures/list.pyi] - -[case testContextManagerWithUnspecifiedArguments] -from contextlib import contextmanager -from typing import Callable, Iterator - -c: Callable[..., Iterator[int]] -reveal_type(c) # N: Revealed type is "def (*Any, **Any) -> typing.Iterator[builtins.int]" -reveal_type(contextmanager(c)) # N: Revealed type is "def (*Any, **Any) -> contextlib.GeneratorContextManager[builtins.int]" -[typing fixtures/typing-medium.pyi] -[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index f1a6f3c77ada..81b8948be14a 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -180,7 +180,9 @@ import nostub # type: ignore[import] from defusedxml import xyz # type: ignore[import] [case testErrorCodeBadIgnore] -import nostub # type: ignore xyz # E: Invalid "type: ignore" comment [syntax] +import nostub # type: ignore xyz # E: Invalid "type: ignore" comment [syntax] \ + # E: Cannot find implementation or library stub for module named "nostub" [import] \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports import nostub # type: ignore[ # E: Invalid "type: ignore" comment [syntax] import nostub # type: ignore[foo # E: Invalid "type: ignore" comment [syntax] import nostub # type: ignore[foo, # E: Invalid "type: ignore" comment [syntax] @@ -207,6 +209,8 @@ def f(x, # type: int # type: ignore[ pass [out] main:2: error: Invalid "type: ignore" comment [syntax] +main:2: error: Cannot find implementation or library stub for module named "nostub" [import] +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports main:3: error: Invalid "type: ignore" comment [syntax] main:4: error: Invalid "type: ignore" comment [syntax] main:5: error: Invalid "type: ignore" comment [syntax] @@ -241,7 +245,8 @@ x: f # E: Function "__main__.f" is not valid as a type [valid-type] \ # N: Perhaps you need "Callable[...]" or a callback protocol? import sys -y: sys # E: Module "sys" is not valid as a type [valid-type] +y: sys # E: Module "sys" is not valid as a type [valid-type] \ + # N: Perhaps you meant to use a protocol matching the module structure? z: y # E: Variable "__main__.y" is not valid as a type [valid-type] \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases [builtins fixtures/tuple.pyi] @@ -250,7 +255,7 @@ z: y # E: Variable "__main__.y" is not valid as a type [valid-type] \ from typing import TypeVar T = TypeVar('T') -def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same Typevar [type-var] +def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same TypeVar [type-var] x = f() # E: Need type annotation for "x" [var-annotated] y = [] # E: Need type annotation for "y" (hint: "y: List[] = ...") [var-annotated] [builtins fixtures/list.pyi] @@ -628,9 +633,6 @@ def g() -> int: [case testErrorCodeIgnoreNamedDefinedNote] x: List[int] # type: ignore[name-defined] -[case testErrorCodeIgnoreMiscNote] -x: [int] # type: ignore[misc] - [case testErrorCodeProtocolProblemsIgnore] from typing_extensions import Protocol @@ -840,19 +842,21 @@ if bad_union: # E: "__main__.bad_union" has type "Union[Foo, object]" of which if not bad_union: # E: "__main__.bad_union" has type "object" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] pass -def f(): - pass -if f: # E: Function "Callable[[], Any]" could always be true in boolean context [truthy-bool] - pass -if not f: # E: Function "Callable[[], Any]" could always be true in boolean context [truthy-bool] - pass -conditional_result = 'foo' if f else 'bar' # E: Function "Callable[[], Any]" could always be true in boolean context [truthy-bool] - lst: List[int] = [] if lst: pass [builtins fixtures/list.pyi] +[case testTruthyFunctions] +# flags: --strict-optional +def f(): + pass +if f: # E: Function "Callable[[], Any]" could always be true in boolean context [truthy-function] + pass +if not f: # E: Function "Callable[[], Any]" could always be true in boolean context [truthy-function] + pass +conditional_result = 'foo' if f else 'bar' # E: Function "Callable[[], Any]" could always be true in boolean context [truthy-function] + [case testNoOverloadImplementation] from typing import overload @@ -916,3 +920,59 @@ def f(d: D, s: str) -> None: d[s] # type: ignore[literal-required] [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testRecommendErrorCode] +# type: ignore[whatever] # E: type ignore with error code is not supported for modules; use `# mypy: disable-error-code=...` [syntax] +1 + "asdf" + +[case testShowErrorCodesInConfig] +# flags: --config-file tmp/mypy.ini +# Test 'show_error_codes = True' in config doesn't raise an exception +var: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] + +[file mypy.ini] +\[mypy] +show_error_codes = True + +[case testErrorCodeUnsafeSuper_no_empty] +# flags: --strict-optional +from abc import abstractmethod + +class Base: + @abstractmethod + def meth(self) -> int: + raise NotImplementedError() +class Sub(Base): + def meth(self) -> int: + return super().meth() # E: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe [safe-super] +[builtins fixtures/exception.pyi] + +[case testDedicatedErrorCodeForEmpty_no_empty] +# flags: --strict-optional +from typing import Optional +def foo() -> int: ... # E: Missing return statement [empty-body] +def bar() -> None: ... +# This is inconsistent with how --warn-no-return behaves in general +# but we want to minimize fallout of finally handling empty bodies. +def baz() -> Optional[int]: ... # OK + +[case testDedicatedErrorCodeTypeAbstract] +import abc +from typing import TypeVar, Type + +class C(abc.ABC): + @abc.abstractmethod + def foo(self) -> None: ... + +T = TypeVar("T") +def test(tp: Type[T]) -> T: ... +test(C) # E: Only concrete class can be given where "Type[C]" is expected [type-abstract] + +[case testUncheckedAnnotationCodeShown] +def f(): + x: int = "no" # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs [annotation-unchecked] + +[case testUncheckedAnnotationSuppressed] +# flags: --disable-error-code=annotation-unchecked +def f(): + x: int = "no" # No warning here diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 577e71d78482..f7aa43d43f3e 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -857,8 +857,8 @@ a[b] a[c] a[1] # E: No overload variant of "__getitem__" of "A" matches argument type "int" \ # N: Possible overload variants: \ - # N: def __getitem__(self, B) -> int \ - # N: def __getitem__(self, C) -> str + # N: def __getitem__(self, B, /) -> int \ + # N: def __getitem__(self, C, /) -> str i, s = None, None # type: (int, str) if int(): @@ -952,6 +952,40 @@ y: Gen[Literal[1]] = assert_type(Gen(1), Gen[Literal[1]]) [builtins fixtures/tuple.pyi] +[case testAssertTypeUncheckedFunction] +from typing import assert_type +from typing_extensions import Literal +def f(): + x = 42 + assert_type(x, Literal[42]) +[out] +main:5: error: Expression is of type "Any", not "Literal[42]" +main:5: note: "assert_type" expects everything to be "Any" in unchecked functions +[builtins fixtures/tuple.pyi] + +[case testAssertTypeUncheckedFunctionWithUntypedCheck] +# flags: --check-untyped-defs +from typing import assert_type +from typing_extensions import Literal +def f(): + x = 42 + assert_type(x, Literal[42]) +[out] +main:6: error: Expression is of type "int", not "Literal[42]" +[builtins fixtures/tuple.pyi] + +[case testAssertTypeNoPromoteUnion] +from typing import Union, assert_type + +Scalar = Union[int, bool, bytes, bytearray] + + +def reduce_it(s: Scalar) -> Scalar: + return s + +assert_type(reduce_it(True), Scalar) +[builtins fixtures/tuple.pyi] + -- None return type -- ---------------- diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index 848d91b1659d..f172a9727d49 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -106,6 +106,7 @@ class C: [builtins fixtures/property.pyi] [case testFastParsePerArgumentAnnotations] +# flags: --implicit-optional class A: pass class B: pass @@ -130,6 +131,7 @@ def f(a, # type: A [out] [case testFastParsePerArgumentAnnotationsWithReturn] +# flags: --implicit-optional class A: pass class B: pass diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index 5b5d49c80708..5a075dd6efef 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -366,6 +366,22 @@ def f() -> NoReturn: # E: Implicit return in function which does not return non_trivial_function = 1 [builtins fixtures/dict.pyi] +[case testNoReturnImplicitReturnCheckInDeferredNode] +# flags: --warn-no-return +from typing import NoReturn + +def exit() -> NoReturn: ... + +def force_forward_reference() -> int: + return 4 + +def f() -> NoReturn: + x + exit() + +x = force_forward_reference() +[builtins fixtures/exception.pyi] + [case testNoReturnNoWarnNoReturn] # flags: --warn-no-return from mypy_extensions import NoReturn @@ -827,6 +843,7 @@ standard.f(None) [file mypy.ini] \[mypy] strict_optional = False +implicit_optional = true \[mypy-optional] strict_optional = True @@ -846,6 +863,7 @@ standard.f(None) [file pyproject.toml] \[tool.mypy] strict_optional = false +implicit_optional = true \[[tool.mypy.overrides]] module = 'optional' strict_optional = true @@ -1595,7 +1613,7 @@ a = 5 [file other_module_2.py] from other_module_1 import a [out] -main:2: error: Module "other_module_2" does not explicitly export attribute "a"; implicit reexport disabled +main:2: error: Module "other_module_2" does not explicitly export attribute "a" [case testNoImplicitReexportRespectsAll] # flags: --no-implicit-reexport @@ -1609,7 +1627,7 @@ from other_module_1 import a, b __all__ = ('b',) [builtins fixtures/tuple.pyi] [out] -main:2: error: Module "other_module_2" does not explicitly export attribute "a"; implicit reexport disabled +main:2: error: Module "other_module_2" does not explicitly export attribute "a" [case testNoImplicitReexportStarConsideredExplicit] # flags: --no-implicit-reexport @@ -1625,7 +1643,7 @@ __all__ = ('b',) [case testNoImplicitReexportGetAttr] # flags: --no-implicit-reexport --python-version 3.7 -from other_module_2 import a # E: Module "other_module_2" does not explicitly export attribute "a"; implicit reexport disabled +from other_module_2 import a # E: Module "other_module_2" does not explicitly export attribute "a" [file other_module_1.py] from typing import Any def __getattr__(name: str) -> Any: ... @@ -1643,7 +1661,7 @@ attr_2 = 6 [file other_module_2.py] from other_module_1 import attr_1, attr_2 [out] -main:2: error: Module "other_module_2" does not explicitly export attribute "attr_1"; implicit reexport disabled +main:2: error: Module "other_module_2" does not explicitly export attribute "attr_1" [case testNoImplicitReexportMypyIni] # flags: --config-file tmp/mypy.ini @@ -1661,7 +1679,7 @@ implicit_reexport = True \[mypy-other_module_2] implicit_reexport = False [out] -main:2: error: Module "other_module_2" has no attribute "a" +main:2: error: Module "other_module_2" does not explicitly export attribute "a" [case testNoImplicitReexportPyProjectTOML] @@ -1682,7 +1700,7 @@ module = 'other_module_2' implicit_reexport = false [out] -main:2: error: Module "other_module_2" has no attribute "a" +main:2: error: Module "other_module_2" does not explicitly export attribute "a" [case testImplicitAnyOKForNoArgs] @@ -2005,12 +2023,12 @@ x = 'should be fine' x.trim() [case testDisableDifferentErrorCode] -# flags: --disable-error-code name-defined --show-error-code +# flags: --disable-error-code name-defined --show-error-codes x = 'should not be fine' x.trim() # E: "str" has no attribute "trim" [attr-defined] [case testDisableMultipleErrorCode] -# flags: --disable-error-code attr-defined --disable-error-code return-value --show-error-code +# flags: --disable-error-code attr-defined --disable-error-code return-value --show-error-codes x = 'should be fine' x.trim() @@ -2020,12 +2038,12 @@ def bad_return_type() -> str: bad_return_type('no args taken!') # E: Too many arguments for "bad_return_type" [call-arg] [case testEnableErrorCode] -# flags: --disable-error-code attr-defined --enable-error-code attr-defined --show-error-code +# flags: --disable-error-code attr-defined --enable-error-code attr-defined --show-error-codes x = 'should be fine' x.trim() # E: "str" has no attribute "trim" [attr-defined] [case testEnableDifferentErrorCode] -# flags: --disable-error-code attr-defined --enable-error-code name-defined --show-error-code +# flags: --disable-error-code attr-defined --enable-error-code name-defined --show-error-codes x = 'should not be fine' x.trim() y.trim() # E: Name "y" is not defined [name-defined] @@ -2036,7 +2054,7 @@ y.trim() # E: Name "y" is not defined [name-defined] --disable-error-code return-value \ --disable-error-code call-arg \ --enable-error-code attr-defined \ - --enable-error-code return-value --show-error-code + --enable-error-code return-value --show-error-codes x = 'should be fine' x.trim() # E: "str" has no attribute "trim" [attr-defined] @@ -2053,3 +2071,75 @@ def f(x): y = 1 f(reveal_type(y)) # E: Call to untyped function "f" in typed context \ # N: Revealed type is "builtins.int" + +[case testPerModuleErrorCodes] +# flags: --config-file tmp/mypy.ini +import tests.foo +import bar +[file bar.py] +x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +[file tests/__init__.py] +[file tests/foo.py] +x = [] # OK +[file mypy.ini] +\[mypy] +strict = True + +\[mypy-tests.*] +allow_untyped_defs = True +allow_untyped_calls = True +disable_error_code = var-annotated + +[case testPerModuleErrorCodesOverride] +# flags: --config-file tmp/mypy.ini +import tests.foo +import bar +[file bar.py] +def foo() -> int: ... +if foo: ... # E: Function "Callable[[], int]" could always be true in boolean context +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) +[file tests/__init__.py] +[file tests/foo.py] +def foo() -> int: ... +if foo: ... # E: Function "Callable[[], int]" could always be true in boolean context +42 + "no" # type: ignore +[file mypy.ini] +\[mypy] +enable_error_code = ignore-without-code, truthy-bool + +\[mypy-tests.*] +disable_error_code = ignore-without-code + +[case testShowErrorCodes] +# flags: --show-error-codes +x: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] + +[case testHideErrorCodes] +# flags: --hide-error-codes +x: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testTypeVarTupleDisabled_no_incomplete] +from typing_extensions import TypeVarTuple +Ts = TypeVarTuple("Ts") # E: "TypeVarTuple" support is experimental, use --enable-incomplete-feature=TypeVarTuple to enable +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleEnabled_no_incomplete] +# flags: --enable-incomplete-feature=TypeVarTuple +from typing_extensions import TypeVarTuple +Ts = TypeVarTuple("Ts") # OK +[builtins fixtures/tuple.pyi] + + +[case testDisableBytearrayPromotion] +# flags: --disable-bytearray-promotion +def f(x: bytes) -> None: ... +f(bytearray(b"asdf")) # E: Argument 1 to "f" has incompatible type "bytearray"; expected "bytes" +f(memoryview(b"asdf")) +[builtins fixtures/primitives.pyi] + +[case testDisableMemoryviewPromotion] +# flags: --disable-memoryview-promotion +def f(x: bytes) -> None: ... +f(bytearray(b"asdf")) +f(memoryview(b"asdf")) # E: Argument 1 to "f" has incompatible type "memoryview"; expected "bytes" +[builtins fixtures/primitives.pyi] diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 32d531ebbe99..bb36b65f35de 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -465,6 +465,7 @@ if int(): [case testCallingFunctionsWithDefaultArgumentValues] +# flags: --implicit-optional --no-strict-optional a, b = None, None # type: (A, B) if int(): @@ -1393,7 +1394,11 @@ x = None # type: Any if x: def f(x: int) -> None: pass else: - def f(x): pass # E: All conditional function variants must have identical signatures + def f(x): pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: int) -> None \ + # N: Redefinition: \ + # N: def f(x: Any) -> Any [case testIncompatibleConditionalFunctionDefinition2] from typing import Any @@ -1401,7 +1406,11 @@ x = None # type: Any if x: def f(x: int) -> None: pass else: - def f(y: int) -> None: pass # E: All conditional function variants must have identical signatures + def f(y: int) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: int) -> None \ + # N: Redefinition: \ + # N: def f(y: int) -> None [case testIncompatibleConditionalFunctionDefinition3] from typing import Any @@ -1409,7 +1418,11 @@ x = None # type: Any if x: def f(x: int) -> None: pass else: - def f(x: int = 0) -> None: pass # E: All conditional function variants must have identical signatures + def f(x: int = 0) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: int) -> None \ + # N: Redefinition: \ + # N: def f(x: int = ...) -> None [case testConditionalFunctionDefinitionUsingDecorator1] from typing import Callable @@ -1467,14 +1480,22 @@ from typing import Any def f(x: str) -> None: pass x = None # type: Any if x: - def f(x: int) -> None: pass # E: All conditional function variants must have identical signatures + def f(x: int) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: str) -> None \ + # N: Redefinition: \ + # N: def f(x: int) -> None [case testConditionalRedefinitionOfAnUnconditionalFunctionDefinition2] from typing import Any def f(x: int) -> None: pass # N: "f" defined here x = None # type: Any if x: - def f(y: int) -> None: pass # E: All conditional function variants must have identical signatures + def f(y: int) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: int) -> None \ + # N: Redefinition: \ + # N: def f(y: int) -> None f(x=1) # The first definition takes precedence. f(y=1) # E: Unexpected keyword argument "y" for "f" @@ -1640,7 +1661,11 @@ class A: if x: def f(self, x: int) -> None: pass else: - def f(self, x): pass # E: All conditional function variants must have identical signatures + def f(self, x): pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(self: A, x: int) -> None \ + # N: Redefinition: \ + # N: def f(self: A, x: Any) -> Any [out] [case testConditionalFunctionDefinitionInTry] @@ -2674,3 +2699,13 @@ class A: def h(self, *args, **kwargs) -> int: pass # OK [builtins fixtures/property.pyi] [out] + +[case testSubtypingUnionGenericBounds] +from typing import Callable, TypeVar, Union, Sequence + +TI = TypeVar("TI", bound=int) +TS = TypeVar("TS", bound=str) + +f: Callable[[Sequence[TI]], None] +g: Callable[[Union[Sequence[TI], Sequence[TS]]], None] +f = g diff --git a/test-data/unit/check-generic-subtyping.test b/test-data/unit/check-generic-subtyping.test index bd1f487bc895..1f06bc7c540a 100644 --- a/test-data/unit/check-generic-subtyping.test +++ b/test-data/unit/check-generic-subtyping.test @@ -1033,3 +1033,21 @@ x2: X2[str, int] reveal_type(iter(x2)) # N: Revealed type is "typing.Iterator[builtins.int]" reveal_type([*x2]) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/dict.pyi] + +[case testIncompatibleVariance] +from typing import TypeVar, Generic +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class A(Generic[T_co]): ... +class B(A[T_contra], Generic[T_contra]): ... # E: Variance of TypeVar "T_contra" incompatible with variance in parent type + +class C(Generic[T_contra]): ... +class D(C[T_co], Generic[T_co]): ... # E: Variance of TypeVar "T_co" incompatible with variance in parent type + +class E(Generic[T]): ... +class F(E[T_co], Generic[T_co]): ... # E: Variance of TypeVar "T_co" incompatible with variance in parent type + +class G(Generic[T]): ... +class H(G[T_contra], Generic[T_contra]): ... # E: Variance of TypeVar "T_contra" incompatible with variance in parent type diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index b8d70d1dae96..b7d98a783a49 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -1557,9 +1557,9 @@ A = TypeVar('A') B = TypeVar('B') def f1(x: A) -> A: ... -def f2(x: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same Typevar +def f2(x: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar def f3(x: B) -> B: ... -def f4(x: int) -> A: ... # E: A function returning TypeVar should receive at least one argument containing the same Typevar +def f4(x: int) -> A: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar y1 = f1 if int(): @@ -1608,8 +1608,8 @@ B = TypeVar('B') T = TypeVar('T') def outer(t: T) -> None: def f1(x: A) -> A: ... - def f2(x: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same Typevar - def f3(x: T) -> A: ... # E: A function returning TypeVar should receive at least one argument containing the same Typevar + def f2(x: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar + def f3(x: T) -> A: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar def f4(x: A) -> T: ... def f5(x: T) -> T: ... @@ -1778,7 +1778,7 @@ from typing import TypeVar A = TypeVar('A') B = TypeVar('B') def f1(x: int, y: A) -> A: ... -def f2(x: int, y: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same Typevar +def f2(x: int, y: A) -> B: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar def f3(x: A, y: B) -> B: ... g = f1 g = f2 diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 44452e2072b3..d4e6779403b4 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -3544,11 +3544,11 @@ class Bar(Baz): pass [file c.py] class Baz: - def __init__(self): + def __init__(self) -> None: self.x = 12 # type: int [file c.py.2] class Baz: - def __init__(self): + def __init__(self) -> None: self.x = 'lol' # type: str [out] [out2] @@ -4600,7 +4600,7 @@ def outer() -> None: [out2] [case testRecursiveAliasImported] - +# flags: --disable-recursive-aliases import a [file a.py] @@ -5730,6 +5730,7 @@ class C: tmp/a.py:2: error: "object" has no attribute "xyz" [case testIncrementalInvalidNamedTupleInUnannotatedFunction] +# flags: --disable-error-code=annotation-unchecked import a [file a.py] @@ -5759,7 +5760,7 @@ class C: [builtins fixtures/tuple.pyi] [case testNamedTupleUpdateNonRecursiveToRecursiveCoarse] -# flags: --enable-recursive-aliases +# flags: --strict-optional import c [file a.py] from b import M @@ -5802,7 +5803,7 @@ tmp/c.py:5: error: Incompatible types in assignment (expression has type "Option tmp/c.py:7: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" [case testTupleTypeUpdateNonRecursiveToRecursiveCoarse] -# flags: --enable-recursive-aliases +# flags: --strict-optional import c [file a.py] from b import M @@ -5835,7 +5836,7 @@ tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") [case testTypeAliasUpdateNonRecursiveToRecursiveCoarse] -# flags: --enable-recursive-aliases +# flags: --strict-optional import c [file a.py] from b import M @@ -5868,7 +5869,7 @@ tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") [case testTypedDictUpdateNonRecursiveToRecursiveCoarse] -# flags: --enable-recursive-aliases +# flags: --strict-optional import c [file a.py] from b import M @@ -5911,6 +5912,38 @@ tmp/c.py:4: note: Revealed type is "TypedDict('a.N', {'r': Union[TypedDict('b.M' tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") tmp/c.py:7: note: Revealed type is "TypedDict('a.N', {'r': Union[TypedDict('b.M', {'r': Union[..., None], 'x': builtins.int}), None], 'x': builtins.int})" +[case testIncrementalAddClassMethodPlugin] +# flags: --config-file tmp/mypy.ini +import b + +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/add_classmethod.py + +[file a.py] +class BaseAddMethod: pass + +class MyClass(BaseAddMethod): + pass + +[file b.py] +import a + +[file b.py.2] +import a + +my_class = a.MyClass() +reveal_type(a.MyClass.foo_classmethod) +reveal_type(a.MyClass.foo_staticmethod) +reveal_type(my_class.foo_classmethod) +reveal_type(my_class.foo_staticmethod) + +[rechecked b] +[out2] +tmp/b.py:4: note: Revealed type is "def ()" +tmp/b.py:5: note: Revealed type is "def (builtins.int) -> builtins.str" +tmp/b.py:6: note: Revealed type is "def ()" +tmp/b.py:7: note: Revealed type is "def (builtins.int) -> builtins.str" [case testGenericNamedTupleSerialization] import b [file a.py] @@ -5957,3 +5990,302 @@ s: str = td["value"] [out] [out2] tmp/b.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testUnpackKwargsSerialize] +import m +[file lib.py] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]): + ... + +[file m.py] +from lib import foo +foo(name='Jennifer', age=38) +[file m.py.2] +from lib import foo +foo(name='Jennifer', age="38") +[builtins fixtures/dict.pyi] +[out] +[out2] +tmp/m.py:2: error: Argument "age" to "foo" has incompatible type "str"; expected "int" + +[case testDisableEnableErrorCodesIncremental] +# flags: --disable-error-code truthy-bool +# flags2: --enable-error-code truthy-bool +class Foo: + pass + +foo = Foo() +if foo: + ... +[out] +[out2] +main:7: error: "__main__.foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context + +[case testModuleAsProtocolImplementationSerialize] +import m +[file m.py] +from typing import Protocol +from lib import C + +class Options(Protocol): + timeout: int + def update(self) -> bool: ... + +def setup(options: Options) -> None: ... +setup(C().config) + +[file lib.py] +import default_config + +class C: + config = default_config + +[file default_config.py] +timeout = 100 +def update() -> bool: ... + +[file default_config.py.2] +timeout = 100 +def update() -> str: ... +[builtins fixtures/module.pyi] +[out] +[out2] +tmp/m.py:9: error: Argument 1 to "setup" has incompatible type Module; expected "Options" +tmp/m.py:9: note: Following member(s) of "Module default_config" have conflicts: +tmp/m.py:9: note: Expected: +tmp/m.py:9: note: def update() -> bool +tmp/m.py:9: note: Got: +tmp/m.py:9: note: def update() -> str + +[case testAbstractBodyTurnsEmptyCoarse] +# flags: --strict-optional +from b import Base + +class Sub(Base): + def meth(self) -> int: + return super().meth() + +[file b.py] +from abc import abstractmethod +class Base: + @abstractmethod + def meth(self) -> int: return 0 + +[file b.py.2] +from abc import abstractmethod +class Base: + @abstractmethod + def meth(self) -> int: ... +[out] +[out2] +main:6: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe + +[case testNoCrashDoubleReexportFunctionEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +def foo(arg: c.C) -> None: pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportBaseEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +class D(c.C): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportMetaEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +import c +class D(metaclass=c.C): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C(type): ... +[file pb2.py.2] +class C(type): ... +[file pb1.py.2] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportTypedDictEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +from typing_extensions import TypedDict +import c +class D(TypedDict): + x: c.C + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[builtins fixtures/dict.pyi] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportTupleEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +from typing import Tuple +import c +class D(Tuple[c.C, int]): pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[builtins fixtures/tuple.pyi] +[out] +[out2] +[out3] + +[case testNoCrashDoubleReexportOverloadEmpty] +import m + +[file m.py] +import f +[file m.py.3] +import f +# modify + +[file f.py] +from typing import Any, overload +import c + +@overload +def foo(arg: int) -> None: ... +@overload +def foo(arg: c.C) -> None: ... +def foo(arg: Any) -> None: + pass + +[file c.py] +from types import C + +[file types.py] +import pb1 +C = pb1.C +[file types.py.2] +import pb1, pb2 +C = pb2.C + +[file pb1.py] +class C: ... +[file pb2.py.2] +class C: ... +[file pb1.py.2] +[out] +[out2] +[out3] diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index 3bab79f5aec2..2e26f54c6e93 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -814,7 +814,7 @@ if int(): from typing import List class A: def __init__(self): - self.x = [] # type: List[int] + self.x = [] # type: List[int] # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs a = A() a.x = [] a.x = [1] diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 04c710af10d1..6767f1c7995c 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -448,7 +448,7 @@ g(None) # Ok f() # Ok because not used to infer local variable type g(a) -def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same Typevar +def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same TypeVar def g(a: T) -> None: pass [out] @@ -1276,6 +1276,21 @@ class A: def h(x: Callable[[], int]) -> None: pass +[case testLambdaJoinWithDynamicConstructor] +from typing import Any, Union + +class Wrapper: + def __init__(self, x: Any) -> None: ... + +def f(cond: bool) -> Any: + f = Wrapper if cond else lambda x: x + reveal_type(f) # N: Revealed type is "def (x: Any) -> Any" + return f(3) + +def g(cond: bool) -> Any: + f = lambda x: x if cond else Wrapper + reveal_type(f) # N: Revealed type is "def (x: Any) -> Any" + return f(3) -- Boolean operators -- ----------------- @@ -1475,9 +1490,8 @@ class A: self.x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") class B(A): - # TODO?: This error is kind of a false positive, unfortunately @property - def x(self) -> List[int]: # E: Signature of "x" incompatible with supertype "A" + def x(self) -> List[int]: # E: Cannot override writeable attribute with read-only property return [123] [builtins fixtures/list.pyi] @@ -2341,7 +2355,7 @@ def main() -> None: [case testDontMarkUnreachableAfterInferenceUninhabited] from typing import TypeVar T = TypeVar('T') -def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same Typevar +def f() -> T: pass # E: A function returning TypeVar should receive at least one argument containing the same TypeVar class C: x = f() # E: Need type annotation for "x" @@ -2394,7 +2408,7 @@ if bool(): [case testLocalPartialTypesWithGlobalInitializedToNone] # flags: --local-partial-types -x = None # E: Need type annotation for "x" +x = None # E: Need type annotation for "x" (hint: "x: Optional[] = ...") def f() -> None: global x @@ -2405,7 +2419,7 @@ reveal_type(x) # N: Revealed type is "None" [case testLocalPartialTypesWithGlobalInitializedToNone2] # flags: --local-partial-types -x = None # E: Need type annotation for "x" +x = None # E: Need type annotation for "x" (hint: "x: Optional[] = ...") def f(): global x @@ -2454,7 +2468,7 @@ reveal_type(a) # N: Revealed type is "builtins.str" [case testLocalPartialTypesWithClassAttributeInitializedToNone] # flags: --local-partial-types class A: - x = None # E: Need type annotation for "x" + x = None # E: Need type annotation for "x" (hint: "x: Optional[] = ...") def f(self) -> None: self.x = 1 @@ -2637,7 +2651,7 @@ from typing import List def f(x): pass class A: - x = None # E: Need type annotation for "x" + x = None # E: Need type annotation for "x" (hint: "x: Optional[] = ...") def f(self, p: List[str]) -> None: self.x = f(p) @@ -2647,7 +2661,7 @@ class A: [case testLocalPartialTypesAccessPartialNoneAttribute] # flags: --local-partial-types class C: - a = None # E: Need type annotation for "a" + a = None # E: Need type annotation for "a" (hint: "a: Optional[] = ...") def f(self, x) -> None: C.a.y # E: Item "None" of "Optional[Any]" has no attribute "y" @@ -2655,7 +2669,7 @@ class C: [case testLocalPartialTypesAccessPartialNoneAttribute2] # flags: --local-partial-types class C: - a = None # E: Need type annotation for "a" + a = None # E: Need type annotation for "a" (hint: "a: Optional[] = ...") def f(self, x) -> None: self.a.y # E: Item "None" of "Optional[Any]" has no attribute "y" @@ -3249,6 +3263,10 @@ if x: reveal_type(x) # N: Revealed type is "builtins.bytes" [builtins fixtures/dict.pyi] +[case testSuggestPep604AnnotationForPartialNone] +# flags: --local-partial-types --python-version 3.10 +x = None # E: Need type annotation for "x" (hint: "x: | None = ...") + [case testTupleContextFromIterable] from typing import TypeVar, Iterable, List, Union @@ -3264,3 +3282,103 @@ from typing import Dict, Iterable, Tuple, Union def foo(x: Union[Tuple[str, Dict[str, int], str], Iterable[object]]) -> None: ... foo(("a", {"a": "b"}, "b")) [builtins fixtures/dict.pyi] + +[case testUseSupertypeAsInferenceContext] +# flags: --strict-optional +from typing import List, Optional + +class B: + x: List[Optional[int]] + +class C(B): + x = [1] + +reveal_type(C().x) # N: Revealed type is "builtins.list[Union[builtins.int, None]]" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextInvalidType] +from typing import List +class P: + x: List[int] +class C(P): + x = ['a'] # E: List item 0 has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartial] +from typing import List + +class A: + x: List[str] + +class B(A): + x = [] + +reveal_type(B().x) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartialError] +class A: + x = ['a', 'b'] + +class B(A): + x = [] + x.append(2) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "str" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartialErrorProperty] +from typing import List + +class P: + @property + def x(self) -> List[int]: ... +class C(P): + x = [] + +C.x.append("no") # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextConflict] +from typing import List +class P: + x: List[int] +class M: + x: List[str] +class C(P, M): + x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +reveal_type(C.x) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi] + +[case testNoPartialInSupertypeAsContext] +class A: + args = {} # E: Need type annotation for "args" (hint: "args: Dict[, ] = ...") + def f(self) -> None: + value = {1: "Hello"} + class B(A): + args = value +[builtins fixtures/dict.pyi] + +[case testInferSimpleLiteralInClassBodyCycle] +import a +[file a.py] +import b +reveal_type(b.B.x) +class A: + x = 42 +[file b.py] +import a +reveal_type(a.A.x) +class B: + x = 42 +[out] +tmp/b.py:2: note: Revealed type is "builtins.int" +tmp/a.py:2: note: Revealed type is "builtins.int" + +[case testUnionTypeCallableInference] +from typing import Callable, Type, TypeVar, Union + +class A: + def __init__(self, x: str) -> None: ... + +T = TypeVar("T") +def type_or_callable(value: T, tp: Union[Type[T], Callable[[int], T]]) -> T: ... +reveal_type(type_or_callable(A("test"), A)) # N: Revealed type is "__main__.A" diff --git a/test-data/unit/check-inline-config.test b/test-data/unit/check-inline-config.test index 578d8eff7ff8..1b2085e33e91 100644 --- a/test-data/unit/check-inline-config.test +++ b/test-data/unit/check-inline-config.test @@ -162,3 +162,51 @@ main:1: error: Unrecognized option: skip_file = True # mypy: strict [out] main:1: error: Setting "strict" not supported in inline configuration: specify it in a configuration file instead, or set individual inline flags (see "mypy -h" for the list of flags enabled in strict mode) + +[case testInlineErrorCodes] +# flags: --strict-optional +# mypy: enable-error-code="ignore-without-code,truthy-bool" +class Foo: + pass + +foo = Foo() +if foo: ... # E: "__main__.foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) + +[case testInlineErrorCodesOverrideConfig] +# flags: --strict-optional --config-file tmp/mypy.ini +import foo +import tests.bar +import tests.baz +[file foo.py] +# mypy: disable-error-code="truthy-bool" +class Foo: + pass + +foo = Foo() +if foo: ... +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) + +[file tests/__init__.py] +[file tests/bar.py] +# mypy: enable-error-code="ignore-without-code" + +def foo() -> int: ... +if foo: ... # E: Function "Callable[[], int]" could always be true in boolean context +42 + "no" # type: ignore # E: "type: ignore" comment without error code (consider "type: ignore[operator]" instead) + +[file tests/baz.py] +# mypy: disable-error-code="truthy-bool" +class Foo: + pass + +foo = Foo() +if foo: ... +42 + "no" # type: ignore + +[file mypy.ini] +\[mypy] +enable_error_code = ignore-without-code, truthy-bool + +\[mypy-tests.*] +disable_error_code = ignore-without-code diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 555e1a568d25..046a4fc43537 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -1321,8 +1321,7 @@ def f(x: Union[A, B]) -> None: f(x) [builtins fixtures/isinstance.pyi] -[case testIsinstanceWithOverlappingPromotionTypes-skip] -# Currently disabled: see https://github.com/python/mypy/issues/6060 for context +[case testIsinstanceWithOverlappingPromotionTypes] from typing import Union class FloatLike: pass @@ -1792,6 +1791,37 @@ issubclass(x, (int, Iterable[int])) # E: Parameterized generics cannot be used [builtins fixtures/isinstance.pyi] [typing fixtures/typing-full.pyi] +[case testIssubclassWithMetaclasses] +class FooMetaclass(type): ... +class Foo(metaclass=FooMetaclass): ... +class Bar: ... + +fm: FooMetaclass +reveal_type(fm) # N: Revealed type is "__main__.FooMetaclass" +if issubclass(fm, Foo): + reveal_type(fm) # N: Revealed type is "Type[__main__.Foo]" +if issubclass(fm, Bar): + reveal_type(fm) # N: Revealed type is "None" +[builtins fixtures/isinstance.pyi] + +[case testIssubclassWithMetaclassesStrictOptional] +# flags: --strict-optional +class FooMetaclass(type): ... +class BarMetaclass(type): ... +class Foo(metaclass=FooMetaclass): ... +class Bar(metaclass=BarMetaclass): ... +class Baz: ... + +fm: FooMetaclass +reveal_type(fm) # N: Revealed type is "__main__.FooMetaclass" +if issubclass(fm, Foo): + reveal_type(fm) # N: Revealed type is "Type[__main__.Foo]" +if issubclass(fm, Bar): + reveal_type(fm) # N: Revealed type is "" +if issubclass(fm, Baz): + reveal_type(fm) # N: Revealed type is "" +[builtins fixtures/isinstance.pyi] + [case testIsinstanceAndNarrowTypeVariable] from typing import TypeVar @@ -2698,3 +2728,189 @@ if type(x) is not C: reveal_type(x) # N: Revealed type is "__main__.D" else: reveal_type(x) # N: Revealed type is "__main__.C" + +[case testHasAttrExistingAttribute] +class C: + x: int +c: C +if hasattr(c, "x"): + reveal_type(c.x) # N: Revealed type is "builtins.int" +else: + # We don't mark this unreachable since people may check for deleted attributes + reveal_type(c.x) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeInstance] +class B: ... +b: B +if hasattr(b, "x"): + reveal_type(b.x) # N: Revealed type is "Any" +else: + b.x # E: "B" has no attribute "x" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeFunction] +def foo(x: int) -> None: ... +if hasattr(foo, "x"): + reveal_type(foo.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeClassObject] +class C: ... +if hasattr(C, "x"): + reveal_type(C.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeTypeType] +from typing import Type +class C: ... +c: Type[C] +if hasattr(c, "x"): + reveal_type(c.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeTypeVar] +from typing import TypeVar + +T = TypeVar("T") +def foo(x: T) -> T: + if hasattr(x, "x"): + reveal_type(x.x) # N: Revealed type is "Any" + return x + else: + return x +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeChained] +class B: ... +b: B +if hasattr(b, "x"): + reveal_type(b.x) # N: Revealed type is "Any" +elif hasattr(b, "y"): + reveal_type(b.y) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeNested] +class A: ... +class B: ... + +x: A +if hasattr(x, "x"): + if isinstance(x, B): + reveal_type(x.x) # N: Revealed type is "Any" + +if hasattr(x, "x") and hasattr(x, "y"): + reveal_type(x.x) # N: Revealed type is "Any" + reveal_type(x.y) # N: Revealed type is "Any" + +if hasattr(x, "x"): + if hasattr(x, "y"): + reveal_type(x.x) # N: Revealed type is "Any" + reveal_type(x.y) # N: Revealed type is "Any" + +if hasattr(x, "x") or hasattr(x, "y"): + x.x # E: "A" has no attribute "x" + x.y # E: "A" has no attribute "y" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrPreciseType] +class A: ... + +x: A +if hasattr(x, "a") and isinstance(x.a, int): + reveal_type(x.a) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeUnion] +from typing import Union + +class A: ... +class B: + x: int + +xu: Union[A, B] +if hasattr(xu, "x"): + reveal_type(xu) # N: Revealed type is "Union[__main__.A, __main__.B]" + reveal_type(xu.x) # N: Revealed type is "Union[Any, builtins.int]" +else: + reveal_type(xu) # N: Revealed type is "__main__.A" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeOuterUnion] +from typing import Union + +class A: ... +class B: ... +xu: Union[A, B] +if isinstance(xu, B): + if hasattr(xu, "x"): + reveal_type(xu.x) # N: Revealed type is "Any" + +if isinstance(xu, B) and hasattr(xu, "x"): + reveal_type(xu.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrDoesntInterfereGetAttr] +class C: + def __getattr__(self, attr: str) -> str: ... + +c: C +if hasattr(c, "foo"): + reveal_type(c.foo) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeLiteral] +from typing import Final +class B: ... +b: B +ATTR: Final = "x" +if hasattr(b, ATTR): + reveal_type(b.x) # N: Revealed type is "Any" +else: + b.x # E: "B" has no attribute "x" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrDeferred] +def foo() -> str: ... + +class Test: + def stream(self) -> None: + if hasattr(self, "_body"): + reveal_type(self._body) # N: Revealed type is "builtins.str" + + def body(self) -> str: + if not hasattr(self, "_body"): + self._body = foo() + return self._body +[builtins fixtures/isinstance.pyi] + +[case testHasAttrModule] +import mod + +if hasattr(mod, "y"): + reveal_type(mod.y) # N: Revealed type is "Any" + reveal_type(mod.x) # N: Revealed type is "builtins.int" +else: + mod.y # E: Module has no attribute "y" + reveal_type(mod.x) # N: Revealed type is "builtins.int" + +if hasattr(mod, "x"): + mod.y # E: Module has no attribute "y" + reveal_type(mod.x) # N: Revealed type is "builtins.int" +else: + mod.y # E: Module has no attribute "y" + reveal_type(mod.x) # N: Revealed type is "builtins.int" + +[file mod.py] +x: int +[builtins fixtures/module.pyi] + +[case testHasAttrDoesntInterfereModuleGetAttr] +import mod + +if hasattr(mod, "y"): + reveal_type(mod.y) # N: Revealed type is "builtins.str" + +[file mod.py] +def __getattr__(attr: str) -> str: ... +[builtins fixtures/module.pyi] diff --git a/test-data/unit/check-kwargs.test b/test-data/unit/check-kwargs.test index 9f8de1265ee7..e59c295b58ac 100644 --- a/test-data/unit/check-kwargs.test +++ b/test-data/unit/check-kwargs.test @@ -24,6 +24,7 @@ class A: pass class B: pass [case testOneOfSeveralOptionalKeywordArguments] +# flags: --implicit-optional import typing def f(a: 'A' = None, b: 'B' = None, c: 'C' = None) -> None: pass f(a=A()) @@ -219,6 +220,7 @@ f(a, **b) [builtins fixtures/dict.pyi] [case testKeywordArgAfterVarArgs] +# flags: --implicit-optional import typing def f(*a: 'A', b: 'B' = None) -> None: pass f() @@ -235,6 +237,7 @@ class B: pass [builtins fixtures/list.pyi] [case testKeywordArgAfterVarArgsWithBothCallerAndCalleeVarArgs] +# flags: --implicit-optional --no-strict-optional from typing import List def f(*a: 'A', b: 'B' = None) -> None: pass a = None # type: List[A] diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index b6eae1da7d84..da8f1570a4f4 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -2918,3 +2918,44 @@ def incorrect_return2() -> Union[Tuple[Literal[True], int], Tuple[Literal[False] else: return (bool(), 'oops') # E: Incompatible return value type (got "Tuple[bool, str]", expected "Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]") [builtins fixtures/bool.pyi] + +[case testLiteralSubtypeContext] +from typing_extensions import Literal + +class A: + foo: Literal['bar', 'spam'] +class B(A): + foo = 'spam' + +reveal_type(B().foo) # N: Revealed type is "Literal['spam']" +[builtins fixtures/tuple.pyi] + +[case testLiteralSubtypeContextNested] +from typing import List +from typing_extensions import Literal + +class A: + foo: List[Literal['bar', 'spam']] +class B(A): + foo = ['spam'] + +reveal_type(B().foo) # N: Revealed type is "builtins.list[Union[Literal['bar'], Literal['spam']]]" +[builtins fixtures/tuple.pyi] + +[case testLiteralSubtypeContextGeneric] +from typing_extensions import Literal +from typing import Generic, List, TypeVar + +T = TypeVar("T", bound=str) + +class B(Generic[T]): + collection: List[T] + word: T + +class C(B[Literal["word"]]): + collection = ["word"] + word = "word" + +reveal_type(C().collection) # N: Revealed type is "builtins.list[Literal['word']]" +reveal_type(C().word) # N: Revealed type is "Literal['word']" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index d83d0470c6b0..a8eced3959e5 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -625,7 +625,11 @@ try: from m import f, g except: def f(x): pass - def g(x): pass # E: All conditional function variants must have identical signatures + def g(x): pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def g(x: Any, y: Any) -> Any \ + # N: Redefinition: \ + # N: def g(x: Any) -> Any [file m.py] def f(x): pass def g(x, y): pass @@ -1668,11 +1672,11 @@ mod_any: Any = m mod_int: int = m # E: Incompatible types in assignment (expression has type Module, variable has type "int") reveal_type(mod_mod) # N: Revealed type is "types.ModuleType" -mod_mod.a # E: Module has no attribute "a" +reveal_type(mod_mod.a) # N: Revealed type is "Any" reveal_type(mod_mod2) # N: Revealed type is "types.ModuleType" -mod_mod2.a # E: Module has no attribute "a" +reveal_type(mod_mod2.a) # N: Revealed type is "Any" reveal_type(mod_mod3) # N: Revealed type is "types.ModuleType" -mod_mod3.a # E: Module has no attribute "a" +reveal_type(mod_mod3.a) # N: Revealed type is "Any" reveal_type(mod_any) # N: Revealed type is "Any" [file m.py] @@ -1732,7 +1736,7 @@ if bool(): else: x = n -x.a # E: Module has no attribute "a" +reveal_type(x.nope) # N: Revealed type is "Any" reveal_type(x.__file__) # N: Revealed type is "builtins.str" [file m.py] @@ -1783,8 +1787,8 @@ m = n # E: Cannot assign multiple modules to name "m" without explicit "types.M [builtins fixtures/module.pyi] [case testNoReExportFromStubs] -from stub import Iterable # E: Module "stub" has no attribute "Iterable" -from stub import D # E: Module "stub" has no attribute "D" +from stub import Iterable # E: Module "stub" does not explicitly export attribute "Iterable" +from stub import D # E: Module "stub" does not explicitly export attribute "D" from stub import C c = C() @@ -1829,7 +1833,7 @@ class C: import stub reveal_type(stub.y) # N: Revealed type is "builtins.int" -reveal_type(stub.z) # E: Module has no attribute "z" \ +reveal_type(stub.z) # E: "Module stub" does not explicitly export attribute "z" \ # N: Revealed type is "Any" [file stub.pyi] @@ -1880,7 +1884,7 @@ class C: from util import mod reveal_type(mod) # N: Revealed type is "def () -> package.mod.mod" -from util import internal_detail # E: Module "util" has no attribute "internal_detail" +from util import internal_detail # E: Module "util" does not explicitly export attribute "internal_detail" [file package/__init__.pyi] from .mod import mod as mod @@ -1895,7 +1899,7 @@ from package import mod as internal_detail [builtins fixtures/module.pyi] [case testNoReExportUnrelatedModule] -from mod2 import unrelated # E: Module "mod2" has no attribute "unrelated" +from mod2 import unrelated # E: Module "mod2" does not explicitly export attribute "unrelated" [file mod1/__init__.pyi] [file mod1/unrelated.pyi] @@ -1906,7 +1910,7 @@ from mod1 import unrelated [builtins fixtures/module.pyi] [case testNoReExportUnrelatedSiblingPrefix] -from pkg.unrel import unrelated # E: Module "pkg.unrel" has no attribute "unrelated" +from pkg.unrel import unrelated # E: Module "pkg.unrel" does not explicitly export attribute "unrelated" [file pkg/__init__.pyi] [file pkg/unrelated.pyi] @@ -1918,10 +1922,10 @@ from pkg import unrelated [case testNoReExportChildStubs] import mod -from mod import C, D # E: Module "mod" has no attribute "C" +from mod import C, D # E: Module "mod" does not explicitly export attribute "C" reveal_type(mod.x) # N: Revealed type is "mod.submod.C" -mod.C # E: Module has no attribute "C" +mod.C # E: "Module mod" does not explicitly export attribute "C" y = mod.D() reveal_type(y.a) # N: Revealed type is "builtins.str" @@ -1936,7 +1940,7 @@ class D: [builtins fixtures/module.pyi] [case testNoReExportNestedStub] -from stub import substub # E: Module "stub" has no attribute "substub" +from stub import substub # E: Module "stub" does not explicitly export attribute "substub" [file stub.pyi] import substub @@ -2661,12 +2665,13 @@ from foo.bar import x x = 0 [case testClassicNotPackage] +# flags: --no-namespace-packages from foo.bar import x [file foo/bar.py] x = 0 [out] -main:1: error: Cannot find implementation or library stub for module named "foo.bar" -main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:2: error: Cannot find implementation or library stub for module named "foo.bar" +main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports [case testNamespacePackage] # flags: --namespace-packages @@ -2882,10 +2887,10 @@ CustomDict = TypedDict( [builtins fixtures/tuple.pyi] [case testNoReExportFromMissingStubs] -from stub import a # E: Module "stub" has no attribute "a" +from stub import a # E: Module "stub" does not explicitly export attribute "a" from stub import b from stub import c # E: Module "stub" has no attribute "c" -from stub import d # E: Module "stub" has no attribute "d" +from stub import d # E: Module "stub" does not explicitly export attribute "d" [file stub.pyi] from mystery import a, b as b, c as d @@ -2904,6 +2909,20 @@ from . import m as m [file p/m.py] [builtins fixtures/list.pyi] +[case testSpecialModulesNameImplicitAttr] +import typing +import builtins +import abc + +reveal_type(abc.__name__) # N: Revealed type is "builtins.str" +reveal_type(builtins.__name__) # N: Revealed type is "builtins.str" +reveal_type(typing.__name__) # N: Revealed type is "builtins.str" + +[case testSpecialAttrsAreAvaliableInClasses] +class Some: + name = __name__ +reveal_type(Some.name) # N: Revealed type is "builtins.str" + [case testReExportAllInStub] from m1 import C from m1 import D # E: Module "m1" has no attribute "D" @@ -3097,10 +3116,15 @@ from google.cloud import x [case testErrorFromGoogleCloud] import google.cloud from google.cloud import x +import google.non_existent +from google.non_existent import x [out] -main:1: error: Cannot find implementation or library stub for module named "google.cloud" +main:1: error: Library stubs not installed for "google.cloud" +main:1: note: Hint: "python3 -m pip install types-google-cloud-ndb" +main:1: note: (or run "mypy --install-types" to install all missing stub packages) main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports main:1: error: Cannot find implementation or library stub for module named "google" +main:3: error: Cannot find implementation or library stub for module named "google.non_existent" [case testMissingSubmoduleOfInstalledStubPackage] import bleach.xyz diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index e4f75f57280c..438e17a6ba0a 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -617,7 +617,7 @@ tmp/b.py:4: note: Revealed type is "Tuple[Any, fallback=a.N]" tmp/b.py:7: note: Revealed type is "Tuple[Any, fallback=a.N]" [case testSimpleSelfReferentialNamedTuple] - +# flags: --disable-recursive-aliases from typing import NamedTuple class MyNamedTuple(NamedTuple): parent: 'MyNamedTuple' # E: Cannot resolve name "MyNamedTuple" (possible cyclic definition) @@ -655,7 +655,7 @@ class B: [out] [case testSelfRefNT1] - +# flags: --disable-recursive-aliases from typing import Tuple, NamedTuple Node = NamedTuple('Node', [ @@ -667,7 +667,7 @@ reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[Any, .. [builtins fixtures/tuple.pyi] [case testSelfRefNT2] - +# flags: --disable-recursive-aliases from typing import Tuple, NamedTuple A = NamedTuple('A', [ @@ -683,7 +683,7 @@ reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[Any, .. [builtins fixtures/tuple.pyi] [case testSelfRefNT3] - +# flags: --disable-recursive-aliases from typing import NamedTuple, Tuple class B(NamedTuple): @@ -703,7 +703,7 @@ reveal_type(lst[0]) # N: Revealed type is "Tuple[builtins.object, builtins.objec [builtins fixtures/tuple.pyi] [case testSelfRefNT4] - +# flags: --disable-recursive-aliases from typing import NamedTuple class B(NamedTuple): @@ -719,7 +719,7 @@ reveal_type(n.y[0]) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [case testSelfRefNT5] - +# flags: --disable-recursive-aliases from typing import NamedTuple B = NamedTuple('B', [ @@ -737,7 +737,7 @@ reveal_type(f) # N: Revealed type is "def (m: Tuple[Any, builtins.int, fallback= [builtins fixtures/tuple.pyi] [case testRecursiveNamedTupleInBases] - +# flags: --disable-recursive-aliases from typing import List, NamedTuple, Union Exp = Union['A', 'B'] # E: Cannot resolve name "Exp" (possible cyclic definition) \ @@ -781,7 +781,7 @@ tp = NamedTuple('tp', [('x', int)]) [out] [case testSubclassOfRecursiveNamedTuple] - +# flags: --disable-recursive-aliases from typing import List, NamedTuple class Command(NamedTuple): @@ -1284,7 +1284,7 @@ from typing import NamedTuple, TypeVar T = TypeVar("T") NT = NamedTuple("NT", [("key", int), ("value", T)]) -reveal_type(NT) # N: Revealed type is "def [T] (key: builtins.int, value: T`-1) -> Tuple[builtins.int, T`-1, fallback=__main__.NT[T`-1]]" +reveal_type(NT) # N: Revealed type is "def [T] (key: builtins.int, value: T`1) -> Tuple[builtins.int, T`1, fallback=__main__.NT[T`1]]" nts: NT[str] reveal_type(nts) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.NT[builtins.str]]" diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 30a41ef86d55..f05e2aaf5c19 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -436,8 +436,8 @@ else: weird_mixture: Union[KeyedTypedDict, KeyedNamedTuple] if weird_mixture["key"] is Key.B: # E: No overload variant of "__getitem__" of "tuple" matches argument type "str" \ # N: Possible overload variants: \ - # N: def __getitem__(self, int) -> Literal[Key.C] \ - # N: def __getitem__(self, slice) -> Tuple[Literal[Key.C], ...] + # N: def __getitem__(self, int, /) -> Literal[Key.C] \ + # N: def __getitem__(self, slice, /) -> Tuple[Literal[Key.C], ...] reveal_type(weird_mixture) # N: Revealed type is "Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]" else: reveal_type(weird_mixture) # N: Revealed type is "Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]" @@ -1249,3 +1249,14 @@ def two_type_vars(x: Union[str, Dict[str, int], Dict[bool, object], int]) -> Non else: reveal_type(x) # N: Revealed type is "builtins.int" [builtins fixtures/dict.pyi] + + +[case testNarrowingWithDef] +from typing import Callable, Optional + +def g() -> None: + foo: Optional[Callable[[], None]] = None + if foo is None: + def foo(): ... + foo() +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-native-int.test b/test-data/unit/check-native-int.test index 14bea5d715c3..24bf0d99b145 100644 --- a/test-data/unit/check-native-int.test +++ b/test-data/unit/check-native-int.test @@ -120,8 +120,10 @@ reveal_type(x) # N: Revealed type is "mypy_extensions.i32" y = 1 if int(): + # We don't narrow an int down to i32, since they have different + # representations. y = i32(1) - reveal_type(y) # N: Revealed type is "mypy_extensions.i32" + reveal_type(y) # N: Revealed type is "builtins.int" reveal_type(y) # N: Revealed type is "builtins.int" [builtins fixtures/dict.pyi] @@ -149,3 +151,36 @@ def fi32(x: i32) -> None: pass reveal_type(meet(ff, fi32)) # N: Revealed type is "" reveal_type(meet(fi32, ff)) # N: Revealed type is "" [builtins fixtures/dict.pyi] + +[case testNativeIntForLoopRange] +from mypy_extensions import i64, i32 + +for a in range(i64(5)): + reveal_type(a) # N: Revealed type is "mypy_extensions.i64" + +for b in range(0, i32(5)): + reveal_type(b) # N: Revealed type is "mypy_extensions.i32" + +for c in range(i64(0), 5): + reveal_type(c) # N: Revealed type is "mypy_extensions.i64" + +for d in range(i64(0), i64(5)): + reveal_type(d) # N: Revealed type is "mypy_extensions.i64" + +for e in range(i64(0), i32(5)): + reveal_type(e) # N: Revealed type is "builtins.int" + +for f in range(0, i64(3), 2): + reveal_type(f) # N: Revealed type is "mypy_extensions.i64" + +n = 5 +for g in range(0, n, i64(2)): + reveal_type(g) # N: Revealed type is "mypy_extensions.i64" +[builtins fixtures/primitives.pyi] + +[case testNativeIntComprehensionRange] +from mypy_extensions import i64, i32 + +reveal_type([a for a in range(i64(5))]) # N: Revealed type is "builtins.list[mypy_extensions.i64]" +[reveal_type(a) for a in range(0, i32(5))] # N: Revealed type is "mypy_extensions.i32" +[builtins fixtures/primitives.pyi] diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index bf612f95b3a2..97cf1ef1494d 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -434,6 +434,7 @@ def main() -> None: x # E: Name "x" is not defined [case testNewAnalyzerCyclicDefinitions] +# flags: --disable-recursive-aliases gx = gy # E: Cannot resolve name "gy" (possible cyclic definition) gy = gx def main() -> None: @@ -1499,6 +1500,7 @@ reveal_type(x[0][0]) # N: Revealed type is "__main__.C" [builtins fixtures/list.pyi] [case testNewAnalyzerAliasToNotReadyDirectBase] +# flags: --disable-recursive-aliases from typing import List x: B @@ -1509,11 +1511,11 @@ reveal_type(x) reveal_type(x[0][0]) [builtins fixtures/list.pyi] [out] -main:3: error: Cannot resolve name "B" (possible cyclic definition) main:4: error: Cannot resolve name "B" (possible cyclic definition) -main:4: error: Cannot resolve name "C" (possible cyclic definition) -main:7: note: Revealed type is "Any" +main:5: error: Cannot resolve name "B" (possible cyclic definition) +main:5: error: Cannot resolve name "C" (possible cyclic definition) main:8: note: Revealed type is "Any" +main:9: note: Revealed type is "Any" [case testNewAnalyzerAliasToNotReadyTwoDeferralsFunction] import a @@ -1532,6 +1534,7 @@ reveal_type(f) # N: Revealed type is "def (x: builtins.list[a.C]) -> builtins.l [builtins fixtures/list.pyi] [case testNewAnalyzerAliasToNotReadyDirectBaseFunction] +# flags: --disable-recursive-aliases import a [file a.py] from typing import List @@ -1864,7 +1867,11 @@ if int(): elif bool(): def f(x: int) -> None: 1() # E: "int" not callable - def g(x: str) -> None: # E: All conditional function variants must have identical signatures + def g(x: str) -> None: # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def g(x: int) -> None \ + # N: Redefinition: \ + # N: def g(x: str) -> None pass else: def f(x: int) -> None: @@ -1881,7 +1888,12 @@ if int(): else: def f(x: A) -> None: 1() # E: "int" not callable - def g(x: str) -> None: # E: All conditional function variants must have identical signatures + def g(x: str) -> None: # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def g(x: A) -> None \ + # N: Redefinition: \ + # N: def g(x: str) -> None + pass reveal_type(g) # N: Revealed type is "def (x: __main__.A)" @@ -2110,6 +2122,7 @@ class B(List[C]): [builtins fixtures/list.pyi] [case testNewAnalyzerNewTypeForwardClassAliasDirect] +# flags: --disable-recursive-aliases from typing import NewType, List x: D @@ -2122,12 +2135,12 @@ class B(D): pass [builtins fixtures/list.pyi] [out] -main:3: error: Cannot resolve name "D" (possible cyclic definition) -main:4: note: Revealed type is "Any" -main:6: error: Cannot resolve name "D" (possible cyclic definition) -main:6: error: Cannot resolve name "C" (possible cyclic definition) -main:7: error: Argument 2 to NewType(...) must be a valid type -main:7: error: Cannot resolve name "B" (possible cyclic definition) +main:4: error: Cannot resolve name "D" (possible cyclic definition) +main:5: note: Revealed type is "Any" +main:7: error: Cannot resolve name "D" (possible cyclic definition) +main:7: error: Cannot resolve name "C" (possible cyclic definition) +main:8: error: Argument 2 to NewType(...) must be a valid type +main:8: error: Cannot resolve name "B" (possible cyclic definition) -- Copied from check-classes.test (tricky corner cases). [case testNewAnalyzerNoCrashForwardRefToBrokenDoubleNewTypeClass] @@ -2144,6 +2157,7 @@ class C: [builtins fixtures/dict.pyi] [case testNewAnalyzerForwardTypeAliasInBase] +# flags: --disable-recursive-aliases from typing import List, Generic, TypeVar, NamedTuple T = TypeVar('T') @@ -2584,6 +2598,7 @@ import n def __getattr__(x): pass [case testNewAnalyzerReportLoopInMRO2] +# flags: --disable-recursive-aliases def f() -> None: class A(A): ... # E: Cannot resolve name "A" (possible cyclic definition) @@ -2948,6 +2963,7 @@ def g() -> None: reveal_type(y) # N: Revealed type is "__main__.G[Any]" [case testNewAnalyzerRedefinedNonlocal] +# flags: --disable-error-code=annotation-unchecked import typing def f(): @@ -2962,7 +2978,7 @@ def g() -> None: def foo() -> None: nonlocal bar - bar = [] # type: typing.List[int] # E: Name "bar" already defined on line 11 + bar = [] # type: typing.List[int] # E: Name "bar" already defined on line 12 [builtins fixtures/list.pyi] [case testNewAnalyzerMoreInvalidTypeVarArgumentsDeferred] diff --git a/test-data/unit/check-optional.test b/test-data/unit/check-optional.test index a0383a35c623..db07290f7b40 100644 --- a/test-data/unit/check-optional.test +++ b/test-data/unit/check-optional.test @@ -127,6 +127,7 @@ def f(x: None) -> None: pass f(None) [case testInferOptionalFromDefaultNone] +# flags: --implicit-optional def f(x: int = None) -> None: x + 1 # E: Unsupported left operand type for + ("None") \ # N: Left operand is of type "Optional[int]" @@ -135,11 +136,14 @@ f(None) [case testNoInferOptionalFromDefaultNone] # flags: --no-implicit-optional -def f(x: int = None) -> None: # E: Incompatible default for argument "x" (default has type "None", argument has type "int") +def f(x: int = None) -> None: # E: Incompatible default for argument "x" (default has type "None", argument has type "int") \ + # N: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True \ + # N: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase pass [out] [case testInferOptionalFromDefaultNoneComment] +# flags: --implicit-optional def f(x=None): # type: (int) -> None x + 1 # E: Unsupported left operand type for + ("None") \ @@ -149,7 +153,9 @@ f(None) [case testNoInferOptionalFromDefaultNoneComment] # flags: --no-implicit-optional -def f(x=None): # E: Incompatible default for argument "x" (default has type "None", argument has type "int") +def f(x=None): # E: Incompatible default for argument "x" (default has type "None", argument has type "int") \ + # N: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True \ + # N: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase # type: (int) -> None pass [out] @@ -396,58 +402,6 @@ reveal_type(None if bool() else 0) # N: Revealed type is "Union[Literal[0]?, No reveal_type([0, None, 0]) # N: Revealed type is "builtins.list[Union[builtins.int, None]]" [builtins fixtures/list.pyi] -[case testOptionalWhitelistSuppressesOptionalErrors] -# flags: --strict-optional-whitelist -import a -import b -[file a.py] -from typing import Optional -x = None # type: Optional[str] -x + "foo" - -[file b.py] -from typing import Optional -x = None # type: Optional[int] -x + 1 - -[builtins fixtures/primitives.pyi] - -[case testOptionalWhitelistPermitsOtherErrors] -# flags: --strict-optional-whitelist -import a -import b -[file a.py] -from typing import Optional -x = None # type: Optional[str] -x + "foo" - -[file b.py] -from typing import Optional -x = None # type: Optional[int] -x + 1 -1 + "foo" -[builtins fixtures/primitives.pyi] -[out] -tmp/b.py:4: error: Unsupported operand types for + ("int" and "str") - -[case testOptionalWhitelistPermitsWhitelistedFiles] -# flags: --strict-optional-whitelist **/a.py -import a -import b -[file a.py] -from typing import Optional -x = None # type: Optional[str] -x + "foo" - -[file b.py] -from typing import Optional -x = None # type: Optional[int] -x + 1 -[builtins fixtures/primitives.pyi] -[out] -tmp/a.py:3: error: Unsupported left operand type for + ("None") -tmp/a.py:3: note: Left operand is of type "Optional[str]" - [case testNoneContextInference] from typing import Dict, List def f() -> List[None]: diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 33ab1a8602be..a5e6cefc2af0 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -906,8 +906,8 @@ B() < B() A() < object() # E: Unsupported operand types for < ("A" and "object") B() < object() # E: No overload variant of "__lt__" of "B" matches argument type "object" \ # N: Possible overload variants: \ - # N: def __lt__(self, B) -> int \ - # N: def __lt__(self, A) -> int + # N: def __lt__(self, B, /) -> int \ + # N: def __lt__(self, A, /) -> int [case testOverloadedForwardMethodAndCallingReverseMethod] from foo import * @@ -925,8 +925,8 @@ A() + 1 A() + B() A() + '' # E: No overload variant of "__add__" of "A" matches argument type "str" \ # N: Possible overload variants: \ - # N: def __add__(self, A) -> int \ - # N: def __add__(self, int) -> int + # N: def __add__(self, A, /) -> int \ + # N: def __add__(self, int, /) -> int [case testOverrideOverloadSwapped] from foo import * @@ -4738,12 +4738,12 @@ reveal_type(actually_b + Other()) # Note [out] main:12: error: Signature of "__add__" incompatible with supertype "A" main:12: note: Superclass: -main:12: note: def __add__(self, A) -> A +main:12: note: def __add__(self, A, /) -> A main:12: note: Subclass: main:12: note: @overload -main:12: note: def __add__(self, Other) -> B +main:12: note: def __add__(self, Other, /) -> B main:12: note: @overload -main:12: note: def __add__(self, A) -> A +main:12: note: def __add__(self, A, /) -> A main:12: note: Overloaded operator methods cannot have wider argument types in overrides main:32: note: Revealed type is "__main__.Other" @@ -5986,10 +5986,10 @@ reveal_type(f2(A())) # E: No overload variant of "f2" matches argument type "A" if True: @overload # E: Single overload definition, multiple required def f3(x: A) -> A: ... + def f3(x): ... if maybe_true: # E: Name "maybe_true" is not defined @overload # E: Single overload definition, multiple required def g3(x: B) -> B: ... - def f3(x): ... reveal_type(f3(A())) # N: Revealed type is "__main__.A" if True: @@ -6467,3 +6467,80 @@ spam: Callable[..., str] = lambda x, y: 'baz' reveal_type(func(spam)) # N: Revealed type is "def (*Any, **Any) -> builtins.str" [builtins fixtures/paramspec.pyi] + +[case testGenericOverloadOverlapWithType] +import m + +[file m.pyi] +from typing import TypeVar, Type, overload, Callable + +T = TypeVar("T", bound=str) +@overload +def foo(x: Type[T] | int) -> int: ... +@overload +def foo(x: Callable[[int], bool]) -> str: ... + +[case testGenericOverloadOverlapWithCollection] +import m + +[file m.pyi] +from typing import TypeVar, Sequence, overload, List + +T = TypeVar("T", bound=str) + +@overload +def foo(x: List[T]) -> str: ... +@overload +def foo(x: Sequence[int]) -> int: ... +[builtins fixtures/list.pyi] + +# Also see `check-python38.test` for similar tests with `/` args: +[case testOverloadPositionalOnlyErrorMessageOldStyle] +from typing import overload + +@overload +def foo(__a: int): ... +@overload +def foo(a: str): ... +def foo(a): ... + +foo(a=1) +[out] +main:9: error: No overload variant of "foo" matches argument type "int" +main:9: note: Possible overload variants: +main:9: note: def foo(int, /) -> Any +main:9: note: def foo(a: str) -> Any + +[case testOverloadUnionGenericBounds] +from typing import overload, TypeVar, Sequence, Union + +class Entity: ... +class Assoc: ... + +E = TypeVar("E", bound=Entity) +A = TypeVar("A", bound=Assoc) + +class Test: + @overload + def foo(self, arg: Sequence[E]) -> None: ... + @overload + def foo(self, arg: Sequence[A]) -> None: ... + def foo(self, arg: Union[Sequence[E], Sequence[A]]) -> None: + ... + +[case testOverloadedStaticMethodOnInstance] +from typing import overload + +class Snafu(object): + @overload + @staticmethod + def snafu(value: bytes) -> bytes: ... + @overload + @staticmethod + def snafu(value: str) -> str: ... + @staticmethod + def snafu(value): + ... +reveal_type(Snafu().snafu('123')) # N: Revealed type is "builtins.str" +reveal_type(Snafu.snafu('123')) # N: Revealed type is "builtins.str" +[builtins fixtures/staticmethod.pyi] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 18192b38dc6c..6f488f108153 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -530,7 +530,7 @@ reveal_type(transform(bar)) # N: Revealed type is "def (builtins.str, *args: bui def expects_int_first(x: Callable[Concatenate[int, P], int]) -> None: ... @expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]" \ - # N: This may be because "one" has arguments named: "x" + # N: This is likely because "one" has named arguments: "x". Consider marking them positional-only def one(x: str) -> int: ... @expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int, NamedArg(int, 'x')], int]" @@ -573,7 +573,7 @@ reveal_type(f(n)) # N: Revealed type is "def (builtins.int, builtins.bytes) -> [builtins fixtures/paramspec.pyi] [case testParamSpecConcatenateNamedArgs] -# flags: --strict-concatenate +# flags: --python-version 3.8 --strict-concatenate # this is one noticeable deviation from PEP but I believe it is for the better from typing_extensions import ParamSpec, Concatenate from typing import Callable, TypeVar @@ -595,12 +595,14 @@ def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: f2(lambda x: 42)(42, x=42) [builtins fixtures/paramspec.pyi] [out] -main:10: error: invalid syntax +main:10: error: invalid syntax; you likely need to run mypy using Python 3.8 or newer [out version>=3.8] main:17: error: Incompatible return value type (got "Callable[[Arg(int, 'x'), **P], R]", expected "Callable[[int, **P], R]") -main:17: note: This may be because "result" has arguments named: "x" +main:17: note: This is likely because "result" has named arguments: "x". Consider marking them positional-only [case testNonStrictParamSpecConcatenateNamedArgs] +# flags: --python-version 3.8 + # this is one noticeable deviation from PEP but I believe it is for the better from typing_extensions import ParamSpec, Concatenate from typing import Callable, TypeVar @@ -622,7 +624,7 @@ def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: f2(lambda x: 42)(42, x=42) [builtins fixtures/paramspec.pyi] [out] -main:9: error: invalid syntax +main:11: error: invalid syntax; you likely need to run mypy using Python 3.8 or newer [out version>=3.8] [case testParamSpecConcatenateWithTypeVar] @@ -644,6 +646,8 @@ reveal_type(n(42)) # N: Revealed type is "None" [builtins fixtures/paramspec.pyi] [case testCallablesAsParameters] +# flags: --python-version 3.8 + # credits to https://github.com/microsoft/pyright/issues/2705 from typing_extensions import ParamSpec, Concatenate from typing import Generic, Callable, Any @@ -661,9 +665,9 @@ reveal_type(abc) bar(abc) [builtins fixtures/paramspec.pyi] [out] -main:11: error: invalid syntax +main:13: error: invalid syntax; you likely need to run mypy using Python 3.8 or newer [out version>=3.8] -main:14: note: Revealed type is "__main__.Foo[[builtins.int, b: builtins.str]]" +main:16: note: Revealed type is "__main__.Foo[[builtins.int, b: builtins.str]]" [case testSolveParamSpecWithSelfType] from typing_extensions import ParamSpec, Concatenate @@ -753,24 +757,24 @@ class C(Generic[P]): # think PhantomData from rust phantom: Optional[Callable[P, None]] - def add_str(self) -> C[Concatenate[int, P]]: - return C[Concatenate[int, P]]() - - def add_int(self) -> C[Concatenate[str, P]]: + def add_str(self) -> C[Concatenate[str, P]]: return C[Concatenate[str, P]]() + def add_int(self) -> C[Concatenate[int, P]]: + return C[Concatenate[int, P]]() + def f(c: C[P]) -> None: reveal_type(c) # N: Revealed type is "__main__.C[P`-1]" n1 = c.add_str() - reveal_type(n1) # N: Revealed type is "__main__.C[[builtins.int, **P`-1]]" + reveal_type(n1) # N: Revealed type is "__main__.C[[builtins.str, **P`-1]]" n2 = n1.add_int() - reveal_type(n2) # N: Revealed type is "__main__.C[[builtins.str, builtins.int, **P`-1]]" + reveal_type(n2) # N: Revealed type is "__main__.C[[builtins.int, builtins.str, **P`-1]]" p1 = c.add_int() - reveal_type(p1) # N: Revealed type is "__main__.C[[builtins.str, **P`-1]]" + reveal_type(p1) # N: Revealed type is "__main__.C[[builtins.int, **P`-1]]" p2 = p1.add_str() - reveal_type(p2) # N: Revealed type is "__main__.C[[builtins.int, builtins.str, **P`-1]]" + reveal_type(p2) # N: Revealed type is "__main__.C[[builtins.str, builtins.int, **P`-1]]" [builtins fixtures/paramspec.pyi] [case testParamSpecLiteralJoin] @@ -1062,7 +1066,7 @@ def callback(func: Callable[[Any], Any]) -> None: ... class Job(Generic[P]): ... @callback -def run_job(job: Job[...]) -> T: ... # E: A function returning TypeVar should receive at least one argument containing the same Typevar +def run_job(job: Job[...]) -> T: ... # E: A function returning TypeVar should receive at least one argument containing the same TypeVar [builtins fixtures/tuple.pyi] [case testTupleAndDictOperationsOnParamSpecArgsAndKwargs] @@ -1092,3 +1096,186 @@ def func(callback: Callable[P, str]) -> Callable[P, str]: return 'baz' return inner [builtins fixtures/paramspec.pyi] + +[case testUnpackingParamsSpecArgsAndKwargs] +from typing import Callable +from typing_extensions import ParamSpec + +P = ParamSpec("P") + +def func(callback: Callable[P, str]) -> Callable[P, str]: + def inner(*args: P.args, **kwargs: P.kwargs) -> str: + a, *b = args + reveal_type(a) # N: Revealed type is "builtins.object" + reveal_type(b) # N: Revealed type is "builtins.list[builtins.object]" + c, *d = kwargs + reveal_type(c) # N: Revealed type is "builtins.str" + reveal_type(d) # N: Revealed type is "builtins.list[builtins.str]" + e = {**kwargs} + reveal_type(e) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]" + return "foo" + return inner +[builtins fixtures/paramspec.pyi] + +[case testParamSpecArgsAndKwargsMissmatch] +from typing import Callable +from typing_extensions import ParamSpec + +P1 = ParamSpec("P1") + +def func(callback: Callable[P1, str]) -> Callable[P1, str]: + def inner( + *args: P1.kwargs, # E: Use "P1.args" for variadic "*" parameter + **kwargs: P1.args, # E: Use "P1.kwargs" for variadic "**" parameter + ) -> str: + return "foo" + return inner +[builtins fixtures/paramspec.pyi] + +[case testParamSpecTestPropAccess] +from typing import Callable +from typing_extensions import ParamSpec + +P1 = ParamSpec("P1") + +def func1(callback: Callable[P1, str]) -> Callable[P1, str]: + def inner( + *args: P1.typo, # E: Use "P1.args" for variadic "*" parameter \ + # E: Name "P1.typo" is not defined + **kwargs: P1.kwargs, + ) -> str: + return "foo" + return inner + +def func2(callback: Callable[P1, str]) -> Callable[P1, str]: + def inner( + *args: P1.args, + **kwargs: P1.__bound__, # E: Use "P1.kwargs" for variadic "**" parameter \ + # E: Name "P1.__bound__" is not defined + ) -> str: + return "foo" + return inner + +def func3(callback: Callable[P1, str]) -> Callable[P1, str]: + def inner( + *args: P1.__bound__, # E: Use "P1.args" for variadic "*" parameter \ + # E: Name "P1.__bound__" is not defined + **kwargs: P1.invalid, # E: Use "P1.kwargs" for variadic "**" parameter \ + # E: Name "P1.invalid" is not defined + ) -> str: + return "foo" + return inner +[builtins fixtures/paramspec.pyi] + + +[case testInvalidParamSpecDefinitionsWithArgsKwargs] +from typing import Callable, ParamSpec + +P = ParamSpec('P') + +def c1(f: Callable[P, int], *args: P.args, **kwargs: P.kwargs) -> int: ... +def c2(f: Callable[P, int]) -> int: ... +def c3(f: Callable[P, int], *args, **kwargs) -> int: ... + +# It is ok to define, +def c4(f: Callable[P, int], *args: int, **kwargs: str) -> int: + # but not ok to call: + f(*args, **kwargs) # E: Argument 1 has incompatible type "*Tuple[int, ...]"; expected "P.args" \ + # E: Argument 2 has incompatible type "**Dict[str, str]"; expected "P.kwargs" + return 1 + +def f1(f: Callable[P, int], *args, **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f2(f: Callable[P, int], *args: P.args, **kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f3(f: Callable[P, int], *args: P.args) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f4(f: Callable[P, int], **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" + +# Error message test: +P1 = ParamSpec('P1') + +def m1(f: Callable[P1, int], *a, **k: P1.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +[builtins fixtures/paramspec.pyi] + + +[case testInvalidParamSpecAndConcatenateDefinitionsWithArgsKwargs] +from typing import Callable, ParamSpec +from typing_extensions import Concatenate + +P = ParamSpec('P') + +def c1(f: Callable[Concatenate[int, P], int], *args: P.args, **kwargs: P.kwargs) -> int: ... +def c2(f: Callable[Concatenate[int, P], int]) -> int: ... +def c3(f: Callable[Concatenate[int, P], int], *args, **kwargs) -> int: ... + +# It is ok to define, +def c4(f: Callable[Concatenate[int, P], int], *args: int, **kwargs: str) -> int: + # but not ok to call: + f(1, *args, **kwargs) # E: Argument 2 has incompatible type "*Tuple[int, ...]"; expected "P.args" \ + # E: Argument 3 has incompatible type "**Dict[str, str]"; expected "P.kwargs" + return 1 + +def f1(f: Callable[Concatenate[int, P], int], *args, **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f2(f: Callable[Concatenate[int, P], int], *args: P.args, **kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f3(f: Callable[Concatenate[int, P], int], *args: P.args) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +def f4(f: Callable[Concatenate[int, P], int], **kwargs: P.kwargs) -> int: ... # E: ParamSpec must have "*args" typed as "P.args" and "**kwargs" typed as "P.kwargs" +[builtins fixtures/paramspec.pyi] + + +[case testValidParamSpecInsideGenericWithoutArgsAndKwargs] +from typing import Callable, ParamSpec, Generic +from typing_extensions import Concatenate + +P = ParamSpec('P') + +class Some(Generic[P]): ... + +def create(s: Some[P], *args: int): ... +def update(s: Some[P], **kwargs: int): ... +def delete(s: Some[P]): ... + +def from_callable1(c: Callable[P, int], *args: int, **kwargs: int) -> Some[P]: ... +def from_callable2(c: Callable[P, int], **kwargs: int) -> Some[P]: ... +def from_callable3(c: Callable[P, int], *args: int) -> Some[P]: ... + +def from_extra1(c: Callable[Concatenate[int, P], int], *args: int, **kwargs: int) -> Some[P]: ... +def from_extra2(c: Callable[Concatenate[int, P], int], **kwargs: int) -> Some[P]: ... +def from_extra3(c: Callable[Concatenate[int, P], int], *args: int) -> Some[P]: ... +[builtins fixtures/paramspec.pyi] + + +[case testUnboundParamSpec] +from typing import Callable, ParamSpec + +P1 = ParamSpec('P1') +P2 = ParamSpec('P2') + +def f0(f: Callable[P1, int], *args: P1.args, **kwargs: P2.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" + +def f1(*args: P1.args): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +def f2(**kwargs: P1.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +def f3(*args: P1.args, **kwargs: int): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +def f4(*args: int, **kwargs: P1.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" + +# Error message is based on the `args` definition: +def f5(*args: P2.args, **kwargs: P1.kwargs): ... # E: ParamSpec must have "*args" typed as "P2.args" and "**kwargs" typed as "P2.kwargs" +def f6(*args: P1.args, **kwargs: P2.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" + +# Multiple `ParamSpec` variables can be found, they should not affect error message: +P3 = ParamSpec('P3') + +def f7(first: Callable[P3, int], *args: P1.args, **kwargs: P2.kwargs): ... # E: ParamSpec must have "*args" typed as "P1.args" and "**kwargs" typed as "P1.kwargs" +def f8(first: Callable[P3, int], *args: P2.args, **kwargs: P1.kwargs): ... # E: ParamSpec must have "*args" typed as "P2.args" and "**kwargs" typed as "P2.kwargs" +[builtins fixtures/paramspec.pyi] + + +[case testArgsKwargsWithoutParamSpecVar] +from typing import Generic, Callable, ParamSpec + +P = ParamSpec('P') + +# This must be allowed: +class Some(Generic[P]): + def call(self, *args: P.args, **kwargs: P.kwargs): ... + +# TODO: this probably should be reported. +def call(*args: P.args, **kwargs: P.kwargs): ... +[builtins fixtures/paramspec.pyi] diff --git a/test-data/unit/check-partially-defined.test b/test-data/unit/check-partially-defined.test new file mode 100644 index 000000000000..d456568c1131 --- /dev/null +++ b/test-data/unit/check-partially-defined.test @@ -0,0 +1,369 @@ +[case testDefinedInOneBranch] +# flags: --enable-error-code partially-defined +if int(): + a = 1 +else: + x = 2 +z = a + 1 # E: Name "a" may be undefined +z = a + 1 # We only report the error on first occurrence. + +[case testElif] +# flags: --enable-error-code partially-defined +if int(): + a = 1 +elif int(): + a = 2 +else: + x = 3 + +z = a + 1 # E: Name "a" may be undefined + +[case testUsedInIf] +# flags: --enable-error-code partially-defined +if int(): + y = 1 +if int(): + x = y # E: Name "y" may be undefined + +[case testDefinedInAllBranches] +# flags: --enable-error-code partially-defined +if int(): + a = 1 +elif int(): + a = 2 +else: + a = 3 +z = a + 1 + +[case testOmittedElse] +# flags: --enable-error-code partially-defined +if int(): + a = 1 +z = a + 1 # E: Name "a" may be undefined + +[case testUpdatedInIf] +# flags: --enable-error-code partially-defined +# Variable a is already defined. Just updating it in an "if" is acceptable. +a = 1 +if int(): + a = 2 +z = a + 1 + +[case testNestedIf] +# flags: --enable-error-code partially-defined +if int(): + if int(): + a = 1 + x = 1 + x = x + 1 + else: + a = 2 + b = a + x # E: Name "x" may be undefined + b = b + 1 +else: + b = 2 +z = a + b # E: Name "a" may be undefined + +[case testVeryNestedIf] +# flags: --enable-error-code partially-defined +if int(): + if int(): + if int(): + a = 1 + else: + a = 2 + x = a + else: + a = 2 + b = a +else: + b = 2 +z = a + b # E: Name "a" may be undefined + +[case testTupleUnpack] +# flags: --enable-error-code partially-defined + +if int(): + (x, y) = (1, 2) +else: + [y, z] = [1, 2] +a = y + x # E: Name "x" may be undefined +a = y + z # E: Name "z" may be undefined + +[case testRedefined] +# flags: --enable-error-code partially-defined +y = 3 +if int(): + if int(): + y = 2 + x = y + 2 +else: + if int(): + y = 2 + x = y + 2 + +x = y + 2 + +[case testGenerator] +# flags: --enable-error-code partially-defined +if int(): + a = 3 +s = [a + 1 for a in [1, 2, 3]] +x = a # E: Name "a" may be undefined + +[case testScope] +# flags: --enable-error-code partially-defined +def foo() -> None: + if int(): + y = 2 + +if int(): + y = 3 +x = y # E: Name "y" may be undefined + +[case testFuncParams] +# flags: --enable-error-code partially-defined +def foo(a: int) -> None: + if int(): + a = 2 + x = a + +[case testWhile] +# flags: --enable-error-code partially-defined +while int(): + x = 1 + +y = x # E: Name "x" may be undefined + +while int(): + z = 1 +else: + z = 2 + +y = z # No error. + +while True: + k = 1 + if int(): + break +y = k # No error. + +[case testForLoop] +# flags: --enable-error-code partially-defined +for x in [1, 2, 3]: + if x: + x = 1 + y = x + z = 1 +else: + z = 2 + +a = z + y # E: Name "y" may be undefined + +[case testReturn] +# flags: --enable-error-code partially-defined +def f1() -> int: + if int(): + x = 1 + else: + return 0 + return x + +def f2() -> int: + if int(): + x = 1 + elif int(): + return 0 + else: + x = 2 + return x + +def f3() -> int: + if int(): + x = 1 + elif int(): + return 0 + else: + y = 2 + return x # E: Name "x" may be undefined + +def f4() -> int: + if int(): + x = 1 + elif int(): + return 0 + else: + return 0 + return x + +def f5() -> int: + # This is a test against crashes. + if int(): + return 1 + if int(): + return 2 + else: + return 3 + return 1 + +[case testAssert] +# flags: --enable-error-code partially-defined +def f1() -> int: + if int(): + x = 1 + else: + assert False, "something something" + return x + +def f2() -> int: + if int(): + x = 1 + elif int(): + assert False + else: + y = 2 + return x # E: Name "x" may be undefined + +[case testRaise] +# flags: --enable-error-code partially-defined +def f1() -> int: + if int(): + x = 1 + else: + raise BaseException("something something") + return x + +def f2() -> int: + if int(): + x = 1 + elif int(): + raise BaseException("something something") + else: + y = 2 + return x # E: Name "x" may be undefined +[builtins fixtures/exception.pyi] + +[case testContinue] +# flags: --enable-error-code partially-defined +def f1() -> int: + while int(): + if int(): + x = 1 + else: + continue + y = x + else: + x = 2 + return x + +def f2() -> int: + while int(): + if int(): + x = 1 + z = 1 + elif int(): + pass + else: + continue + y = x # E: Name "x" may be undefined + else: + x = 2 + z = 2 + return z # E: Name "z" may be undefined + +def f3() -> None: + while True: + if int(): + x = 2 + elif int(): + continue + else: + continue + y = x + +[case testBreak] +# flags: --enable-error-code partially-defined +def f1() -> None: + while int(): + if int(): + x = 1 + else: + break + y = x # No error -- x is always defined. + +def f2() -> None: + while int(): + if int(): + x = 1 + elif int(): + pass + else: + break + y = x # E: Name "x" may be undefined + +def f3() -> None: + while int(): + x = 1 + while int(): + if int(): + x = 2 + else: + break + y = x + z = x # E: Name "x" may be undefined + +[case testNoReturn] +# flags: --enable-error-code partially-defined + +from typing import NoReturn +def fail() -> NoReturn: + assert False + +def f() -> None: + if int(): + x = 1 + elif int(): + x = 2 + y = 3 + else: + # This has a NoReturn type, so we can skip it. + fail() + z = y # E: Name "y" may be undefined + z = x + +[case testDictComprehension] +# flags: --enable-error-code partially-defined + +def f() -> None: + for _ in [1, 2]: + key = 2 + val = 2 + + x = ( + key, # E: Name "key" may be undefined + val, # E: Name "val" may be undefined + ) + + d = [(0, "a"), (1, "b")] + {val: key for key, val in d} +[builtins fixtures/dict.pyi] + +[case testWithStmt] +# flags: --enable-error-code partially-defined +from contextlib import contextmanager + +@contextmanager +def ctx(*args): + yield 1 + +def f() -> None: + if int(): + a = b = 1 + x = 1 + + with ctx() as a, ctx(a) as b, ctx(x) as x: # E: Name "x" may be undefined + c = a + c = b + d = a + d = b +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 3dfa30273e6f..8cdfd2a3e0d9 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1190,6 +1190,25 @@ z4 = y4 # E: Incompatible types in assignment (expression has type "PP", variabl # N: Protocol member PPS.attr expected settable variable, got read-only attribute [builtins fixtures/property.pyi] +[case testFinalAttributeProtocol] +from typing import Protocol, Final + +class P(Protocol): + x: int + +class C: + def __init__(self, x: int) -> None: + self.x = x +class CF: + def __init__(self, x: int) -> None: + self.x: Final = x + +x: P +y: P +x = C(42) +y = CF(42) # E: Incompatible types in assignment (expression has type "CF", variable has type "P") \ + # N: Protocol member P.x expected settable variable, got read-only attribute + [case testStaticAndClassMethodsInProtocols] from typing import Protocol, Type, TypeVar @@ -2896,12 +2915,12 @@ c: Lst3[Str] f(Lst3(c)) # E: Argument 1 to "f" has incompatible type "Lst3[Lst3[Str]]"; expected "GetItem[GetItem[Str]]" \ # N: Following member(s) of "Lst3[Lst3[Str]]" have conflicts: \ # N: Expected: \ -# N: def __getitem__(self, int) -> GetItem[Str] \ +# N: def __getitem__(self, int, /) -> GetItem[Str] \ # N: Got: \ # N: @overload \ -# N: def __getitem__(self, slice) -> Lst3[Lst3[Str]] \ +# N: def __getitem__(self, slice, /) -> Lst3[Lst3[Str]] \ # N: @overload \ -# N: def __getitem__(self, bool) -> Lst3[Str] +# N: def __getitem__(self, bool, /) -> Lst3[Str] [builtins fixtures/list.pyi] [typing fixtures/typing-full.pyi] @@ -3086,14 +3105,14 @@ class NoneCompatible(Protocol): class A(NoneCompatible): ... A() # E: Cannot instantiate abstract class "A" with abstract attributes "f", "g", "h", "i" and "j" \ - # N: The following methods were marked implicitly abstract because they have empty function bodies: "f", "g", "h", "i" and "j". If they are not meant to be abstract, explicitly return None. + # N: The following methods were marked implicitly abstract because they have empty function bodies: "f", "g", "h", "i" and "j". If they are not meant to be abstract, explicitly `return` or `return None`. class NoneCompatible2(Protocol): def f(self, x: int): ... class B(NoneCompatible2): ... B() # E: Cannot instantiate abstract class "B" with abstract attribute "f" \ - # N: The following method was marked implicitly abstract because it has an empty function body: "f". If it is not meant to be abstract, explicitly return None. + # N: "f" is implicitly abstract because it has an empty function body. If it is not meant to be abstract, explicitly `return` or `return None`. class NoneCompatible3(Protocol): @abstractmethod @@ -3118,3 +3137,653 @@ class P(Protocol): class A(P): ... A() # E: Cannot instantiate abstract class "A" with abstract attribute "f" + +[case testProtocolWithNestedClass] +from typing import TypeVar, Protocol + +class Template(Protocol): + var: int + class Meta: ... + +class B: + var: int + class Meta: ... +class C: + var: int + class Meta(Template.Meta): ... + +def foo(t: Template) -> None: ... +foo(B()) # E: Argument 1 to "foo" has incompatible type "B"; expected "Template" \ + # N: Following member(s) of "B" have conflicts: \ + # N: Meta: expected "Type[__main__.Template.Meta]", got "Type[__main__.B.Meta]" +foo(C()) # OK + +[case testProtocolClassObjectAttribute] +from typing import ClassVar, Protocol + +class P(Protocol): + foo: int + +class A: + foo = 42 +class B: + foo: ClassVar[int] +class C: + foo: ClassVar[str] +class D: + foo: int + +def test(arg: P) -> None: ... +test(A) # OK +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: foo: expected "int", got "str" +test(D) # E: Argument 1 to "test" has incompatible type "Type[D]"; expected "P" \ + # N: Only class variables allowed for class object access on protocols, foo is an instance variable of "D" + +[case testProtocolClassObjectClassVarRejected] +from typing import ClassVar, Protocol + +class P(Protocol): + foo: ClassVar[int] + +class B: + foo: ClassVar[int] + +def test(arg: P) -> None: ... +test(B) # E: Argument 1 to "test" has incompatible type "Type[B]"; expected "P" \ + # N: ClassVar protocol member P.foo can never be matched by a class object + +[case testProtocolClassObjectPropertyRejected] +from typing import ClassVar, Protocol + +class P(Protocol): + @property + def foo(self) -> int: ... + +class B: + @property + def foo(self) -> int: ... +class C: + foo: int +class D: + foo: ClassVar[int] + +def test(arg: P) -> None: ... +# TODO: skip type mismatch diagnostics in this case. +test(B) # E: Argument 1 to "test" has incompatible type "Type[B]"; expected "P" \ + # N: Following member(s) of "B" have conflicts: \ + # N: foo: expected "int", got "Callable[[B], int]" \ + # N: Only class variables allowed for class object access on protocols, foo is an instance variable of "B" +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Only class variables allowed for class object access on protocols, foo is an instance variable of "C" +test(D) # OK +[builtins fixtures/property.pyi] + +[case testProtocolClassObjectInstanceMethod] +from typing import Any, Protocol + +class P(Protocol): + def foo(self, obj: Any) -> int: ... + +class B: + def foo(self) -> int: ... +class C: + def foo(self) -> str: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo(obj: Any) -> int \ + # N: Got: \ + # N: def foo(self: C) -> str + +[case testProtocolClassObjectInstanceMethodArg] +from typing import Any, Protocol + +class P(Protocol): + def foo(self, obj: B) -> int: ... + +class B: + def foo(self) -> int: ... +class C: + def foo(self) -> int: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo(obj: B) -> int \ + # N: Got: \ + # N: def foo(self: C) -> int + +[case testProtocolClassObjectInstanceMethodOverloaded] +from typing import Any, Protocol, overload + +class P(Protocol): + @overload + def foo(self, obj: Any, arg: int) -> int: ... + @overload + def foo(self, obj: Any, arg: str) -> str: ... + +class B: + @overload + def foo(self, arg: int) -> int: ... + @overload + def foo(self, arg: str) -> str: ... + def foo(self, arg: Any) -> Any: + ... + +class C: + @overload + def foo(self, arg: int) -> int: ... + @overload + def foo(self, arg: str) -> int: ... + def foo(self, arg: Any) -> Any: + ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: @overload \ + # N: def foo(obj: Any, arg: int) -> int \ + # N: @overload \ + # N: def foo(obj: Any, arg: str) -> str \ + # N: Got: \ + # N: @overload \ + # N: def foo(self: C, arg: int) -> int \ + # N: @overload \ + # N: def foo(self: C, arg: str) -> int + +[case testProtocolClassObjectClassMethod] +from typing import Protocol + +class P(Protocol): + def foo(self) -> int: ... + +class B: + @classmethod + def foo(cls) -> int: ... +class C: + @classmethod + def foo(cls) -> str: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> int \ + # N: Got: \ + # N: def foo() -> str +[builtins fixtures/classmethod.pyi] + +[case testProtocolClassObjectStaticMethod] +from typing import Protocol + +class P(Protocol): + def foo(self) -> int: ... + +class B: + @staticmethod + def foo() -> int: ... +class C: + @staticmethod + def foo() -> str: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> int \ + # N: Got: \ + # N: def foo() -> str +[builtins fixtures/staticmethod.pyi] + +[case testProtocolClassObjectGenericInstanceMethod] +from typing import Any, Protocol, Generic, List, TypeVar + +class P(Protocol): + def foo(self, obj: Any) -> List[int]: ... + +T = TypeVar("T") +class A(Generic[T]): + def foo(self) -> T: ... +class AA(A[List[T]]): ... + +class B(AA[int]): ... +class C(AA[str]): ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo(obj: Any) -> List[int] \ + # N: Got: \ + # N: def foo(self: A[List[str]]) -> List[str] +[builtins fixtures/list.pyi] + +[case testProtocolClassObjectGenericClassMethod] +from typing import Any, Protocol, Generic, List, TypeVar + +class P(Protocol): + def foo(self) -> List[int]: ... + +T = TypeVar("T") +class A(Generic[T]): + @classmethod + def foo(self) -> T: ... +class AA(A[List[T]]): ... + +class B(AA[int]): ... +class C(AA[str]): ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> List[int] \ + # N: Got: \ + # N: def foo() -> List[str] +[builtins fixtures/isinstancelist.pyi] + +[case testProtocolClassObjectSelfTypeInstanceMethod] +from typing import Protocol, TypeVar, Union + +T = TypeVar("T") +class P(Protocol): + def foo(self, arg: T) -> T: ... + +class B: + def foo(self: T) -> T: ... +class C: + def foo(self: T) -> Union[T, int]: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def [T] foo(arg: T) -> T \ + # N: Got: \ + # N: def [T] foo(self: T) -> Union[T, int] + +[case testProtocolClassObjectSelfTypeClassMethod] +from typing import Protocol, Type, TypeVar + +T = TypeVar("T") +class P(Protocol): + def foo(self) -> B: ... + +class B: + @classmethod + def foo(cls: Type[T]) -> T: ... +class C: + @classmethod + def foo(cls: Type[T]) -> T: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> B \ + # N: Got: \ + # N: def foo() -> C +[builtins fixtures/classmethod.pyi] + +[case testProtocolClassObjectAttributeAndCall] +from typing import Any, ClassVar, Protocol + +class P(Protocol): + foo: int + def __call__(self, x: int, y: int) -> Any: ... + +class B: + foo: ClassVar[int] + def __init__(self, x: int, y: int) -> None: ... +class C: + foo: ClassVar[int] + def __init__(self, x: int, y: str) -> None: ... + +def test(arg: P) -> None: ... +test(B) # OK +test(C) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: "C" has constructor incompatible with "__call__" of "P" + +[case testProtocolTypeTypeAttribute] +from typing import ClassVar, Protocol, Type + +class P(Protocol): + foo: int + +class A: + foo = 42 +class B: + foo: ClassVar[int] +class C: + foo: ClassVar[str] +class D: + foo: int + +def test(arg: P) -> None: ... +a: Type[A] +b: Type[B] +c: Type[C] +d: Type[D] +test(a) # OK +test(b) # OK +test(c) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: foo: expected "int", got "str" +test(d) # E: Argument 1 to "test" has incompatible type "Type[D]"; expected "P" \ + # N: Only class variables allowed for class object access on protocols, foo is an instance variable of "D" + +[case testProtocolTypeTypeInstanceMethod] +from typing import Any, Protocol, Type + +class P(Protocol): + def foo(self, cls: Any) -> int: ... + +class B: + def foo(self) -> int: ... +class C: + def foo(self) -> str: ... + +def test(arg: P) -> None: ... +b: Type[B] +c: Type[C] +test(b) # OK +test(c) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo(cls: Any) -> int \ + # N: Got: \ + # N: def foo(self: C) -> str + +[case testProtocolTypeTypeClassMethod] +from typing import Protocol, Type + +class P(Protocol): + def foo(self) -> int: ... + +class B: + @classmethod + def foo(cls) -> int: ... +class C: + @classmethod + def foo(cls) -> str: ... + +def test(arg: P) -> None: ... +b: Type[B] +c: Type[C] +test(b) # OK +test(c) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def foo() -> int \ + # N: Got: \ + # N: def foo() -> str +[builtins fixtures/classmethod.pyi] + +[case testProtocolTypeTypeSelfTypeInstanceMethod] +from typing import Protocol, Type, TypeVar, Union + +T = TypeVar("T") +class P(Protocol): + def foo(self, arg: T) -> T: ... + +class B: + def foo(self: T) -> T: ... +class C: + def foo(self: T) -> Union[T, int]: ... + +def test(arg: P) -> None: ... +b: Type[B] +c: Type[C] +test(b) # OK +test(c) # E: Argument 1 to "test" has incompatible type "Type[C]"; expected "P" \ + # N: Following member(s) of "C" have conflicts: \ + # N: Expected: \ + # N: def [T] foo(arg: T) -> T \ + # N: Got: \ + # N: def [T] foo(self: T) -> Union[T, int] + +[case testProtocolClassObjectInference] +from typing import Any, Protocol, TypeVar + +T = TypeVar("T", contravariant=True) +class P(Protocol[T]): + def foo(self, obj: T) -> int: ... + +class B: + def foo(self) -> int: ... + +S = TypeVar("S") +def test(arg: P[S]) -> S: ... +reveal_type(test(B)) # N: Revealed type is "__main__.B" + +[case testProtocolTypeTypeInference] +from typing import Any, Protocol, TypeVar, Type + +T = TypeVar("T", contravariant=True) +class P(Protocol[T]): + def foo(self, obj: T) -> int: ... + +class B: + def foo(self) -> int: ... + +S = TypeVar("S") +def test(arg: P[S]) -> S: ... +b: Type[B] +reveal_type(test(b)) # N: Revealed type is "__main__.B" + +[case testTypeAliasInProtocolBody] +from typing import Protocol, List + +class P(Protocol): + x = List[str] # E: Type aliases are prohibited in protocol bodies \ + # N: Use variable annotation syntax to define protocol members + +class C: + x: int +def foo(x: P) -> None: ... +foo(C()) # No extra error here +[builtins fixtures/list.pyi] + +[case testTypeVarInProtocolBody] +from typing import Protocol, TypeVar + +class C(Protocol): + T = TypeVar('T') + def __call__(self, t: T) -> T: ... + +def f_bad(t: int) -> int: + return t + +S = TypeVar("S") +def f_good(t: S) -> S: + return t + +g: C = f_bad # E: Incompatible types in assignment (expression has type "Callable[[int], int]", variable has type "C") \ + # N: "C.__call__" has type "Callable[[Arg(T, 't')], T]" +g = f_good # OK + +[case testModuleAsProtocolImplementation] +import default_config +import bad_config_1 +import bad_config_2 +import bad_config_3 +from typing import Protocol + +class Options(Protocol): + timeout: int + one_flag: bool + other_flag: bool + def update(self) -> bool: ... + +def setup(options: Options) -> None: ... +setup(default_config) # OK +setup(bad_config_1) # E: Argument 1 to "setup" has incompatible type Module; expected "Options" \ + # N: "ModuleType" is missing following "Options" protocol member: \ + # N: timeout +setup(bad_config_2) # E: Argument 1 to "setup" has incompatible type Module; expected "Options" \ + # N: Following member(s) of "Module bad_config_2" have conflicts: \ + # N: one_flag: expected "bool", got "int" +setup(bad_config_3) # E: Argument 1 to "setup" has incompatible type Module; expected "Options" \ + # N: Following member(s) of "Module bad_config_3" have conflicts: \ + # N: Expected: \ + # N: def update() -> bool \ + # N: Got: \ + # N: def update(obj: Any) -> bool + +[file default_config.py] +timeout = 100 +one_flag = True +other_flag = False +def update() -> bool: ... + +[file bad_config_1.py] +one_flag = True +other_flag = False +def update() -> bool: ... + +[file bad_config_2.py] +timeout = 100 +one_flag = 42 +other_flag = False +def update() -> bool: ... + +[file bad_config_3.py] +timeout = 100 +one_flag = True +other_flag = False +def update(obj) -> bool: ... +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationInference] +import default_config +from typing import Protocol, TypeVar + +T = TypeVar("T", covariant=True) +class Options(Protocol[T]): + timeout: int + one_flag: bool + other_flag: bool + def update(self) -> T: ... + +def setup(options: Options[T]) -> T: ... +reveal_type(setup(default_config)) # N: Revealed type is "builtins.str" + +[file default_config.py] +timeout = 100 +one_flag = True +other_flag = False +def update() -> str: ... +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationClassObject] +import runner +import bad_runner +from typing import Callable, Protocol + +class Runner(Protocol): + @property + def Run(self) -> Callable[[int], Result]: ... + +class Result(Protocol): + value: int + +def run(x: Runner) -> None: ... +run(runner) # OK +run(bad_runner) # E: Argument 1 to "run" has incompatible type Module; expected "Runner" \ + # N: Following member(s) of "Module bad_runner" have conflicts: \ + # N: Expected: \ + # N: def (int, /) -> Result \ + # N: Got: \ + # N: def __init__(arg: str) -> Run + +[file runner.py] +class Run: + value: int + def __init__(self, arg: int) -> None: ... + +[file bad_runner.py] +class Run: + value: int + def __init__(self, arg: str) -> None: ... +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationTypeAlias] +import runner +import bad_runner +from typing import Callable, Protocol + +class Runner(Protocol): + @property + def run(self) -> Callable[[int], Result]: ... + +class Result(Protocol): + value: int + +def run(x: Runner) -> None: ... +run(runner) # OK +run(bad_runner) # E: Argument 1 to "run" has incompatible type Module; expected "Runner" \ + # N: Following member(s) of "Module bad_runner" have conflicts: \ + # N: Expected: \ + # N: def (int, /) -> Result \ + # N: Got: \ + # N: def __init__(arg: str) -> Run + +[file runner.py] +class Run: + value: int + def __init__(self, arg: int) -> None: ... +run = Run + +[file bad_runner.py] +class Run: + value: int + def __init__(self, arg: str) -> None: ... +run = Run +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationClassVar] +from typing import ClassVar, Protocol +import mod + +class My(Protocol): + x: ClassVar[int] + +def test(mod: My) -> None: ... +test(mod=mod) # E: Argument "mod" to "test" has incompatible type Module; expected "My" \ + # N: Protocol member My.x expected class variable, got instance variable +[file mod.py] +x: int +[builtins fixtures/module.pyi] + +[case testModuleAsProtocolImplementationFinal] +from typing import Protocol +import some_module + +class My(Protocol): + a: int + +def func(arg: My) -> None: ... +func(some_module) # E: Argument 1 to "func" has incompatible type Module; expected "My" \ + # N: Protocol member My.a expected settable variable, got read-only attribute + +[file some_module.py] +from typing_extensions import Final + +a: Final = 1 +[builtins fixtures/module.pyi] diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 0003ad2601e0..1548d5dadcfd 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -317,6 +317,21 @@ match x: case [str()]: pass +[case testMatchSequencePatternWithInvalidClassPattern] +class Example: + __match_args__ = ("value",) + def __init__(self, value: str) -> None: + self.value = value + +SubClass: type[Example] + +match [SubClass("a"), SubClass("b")]: + case [SubClass(value), *rest]: # E: Expected type in class pattern; found "Type[__main__.Example]" + reveal_type(value) # E: Cannot determine type of "value" \ + # N: Revealed type is "Any" + reveal_type(rest) # N: Revealed type is "builtins.list[__main__.Example]" +[builtins fixtures/tuple.pyi] + [case testMatchSequenceUnion-skip] from typing import List, Union m: Union[List[List[str]], str] @@ -1600,3 +1615,176 @@ def foo(x: NoneType): # E: NoneType should not be used as a type, please use Non reveal_type(x) # N: Revealed type is "None" [builtins fixtures/tuple.pyi] + +[case testMatchTupleInstanceUnionNoCrash] +from typing import Union + +def func(e: Union[str, tuple[str]]) -> None: + match e: + case (a,) if isinstance(a, str): + reveal_type(a) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + +[case testMatchTupleOptionalNoCrash] +# flags: --strict-optional +foo: tuple[int] | None +match foo: + case x,: + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + +[case testMatchUnionTwoTuplesNoCrash] +var: tuple[int, int] | tuple[str, str] + +# TODO: we can infer better here. +match var: + case (42, a): + reveal_type(a) # N: Revealed type is "Union[builtins.int, builtins.str]" + case ("yes", b): + reveal_type(b) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testMatchNamedAndKeywordsAreTheSame] +from typing import Generic, TypeVar, Union +from typing_extensions import Final +from dataclasses import dataclass + +T = TypeVar("T") + +class Regular: + x: str + y: int + __match_args__ = ("x",) +class ReveresedOrder: + x: int + y: str + __match_args__ = ("y",) +class GenericRegular(Generic[T]): + x: T + __match_args__ = ("x",) +class GenericWithFinal(Generic[T]): + x: T + __match_args__: Final = ("x",) +class RegularSubtype(GenericRegular[str]): ... + +@dataclass +class GenericDataclass(Generic[T]): + x: T + +input_arg: Union[ + Regular, + ReveresedOrder, + GenericRegular[str], + GenericWithFinal[str], + RegularSubtype, + GenericDataclass[str], +] + +# Positional: +match input_arg: + case Regular(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case ReveresedOrder(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericWithFinal(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case RegularSubtype(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericRegular(a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericDataclass(a): + reveal_type(a) # N: Revealed type is "builtins.str" + +# Keywords: +match input_arg: + case Regular(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" + case ReveresedOrder(x=b): # Order is different + reveal_type(b) # N: Revealed type is "builtins.int" + case GenericWithFinal(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" + case RegularSubtype(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericRegular(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" + case GenericDataclass(x=a): + reveal_type(a) # N: Revealed type is "builtins.str" +[builtins fixtures/dataclasses.pyi] + +[case testMatchValueConstrainedTypeVar] +from typing import TypeVar, Iterable + +S = TypeVar("S", int, str) + +def my_func(pairs: Iterable[tuple[S, S]]) -> None: + for pair in pairs: + reveal_type(pair) # N: Revealed type is "Tuple[builtins.int, builtins.int]" \ + # N: Revealed type is "Tuple[builtins.str, builtins.str]" + match pair: + case _: + reveal_type(pair) # N: Revealed type is "Tuple[builtins.int, builtins.int]" \ + # N: Revealed type is "Tuple[builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testPartiallyDefinedMatch] +# flags: --enable-error-code partially-defined +def f0(x: int | str) -> int: + match x: + case int(): + y = 1 + return y # E: Name "y" may be undefined + +def f1(a: object) -> None: + match a: + case [y]: pass + case _: + y = 1 + x = 2 + z = y + z = x # E: Name "x" may be undefined + +def f2(a: object) -> None: + match a: + case [[y] as x]: pass + case {"k1": 1, "k2": x, "k3": y}: pass + case [0, *x]: + y = 2 + case _: + y = 1 + x = [2] + z = x + z = y + +def f3(a: object) -> None: + y = 1 + match a: + case [x]: + y = 2 + # Note the missing `case _:` + z = x # E: Name "x" may be undefined + z = y + +def f4(a: object) -> None: + y = 1 + match a: + case [x]: + y = 2 + case _: + assert False, "unsupported" + z = x + z = y + +def f5(a: object) -> None: + match a: + case tuple(x): pass + case _: + return + y = x + +def f6(a: object) -> None: + if int(): + y = 1 + match a: + case _ if y is not None: # E: Name "y" may be undefined + pass +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-python311.test b/test-data/unit/check-python311.test new file mode 100644 index 000000000000..9bf62b0c489d --- /dev/null +++ b/test-data/unit/check-python311.test @@ -0,0 +1,53 @@ +[case testTryStarSimple] +try: + pass +except* Exception as e: + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[builtins.Exception]" +[builtins fixtures/exception.pyi] + +[case testTryStarMultiple] +try: + pass +except* Exception as e: + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[builtins.Exception]" +except* RuntimeError as e: + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[builtins.RuntimeError]" +[builtins fixtures/exception.pyi] + +[case testTryStarBase] +try: + pass +except* BaseException as e: + reveal_type(e) # N: Revealed type is "builtins.BaseExceptionGroup[builtins.BaseException]" +[builtins fixtures/exception.pyi] + +[case testTryStarTuple] +class Custom(Exception): ... + +try: + pass +except* (RuntimeError, Custom) as e: + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[Union[builtins.RuntimeError, __main__.Custom]]" +[builtins fixtures/exception.pyi] + +[case testTryStarInvalidType] +class Bad: ... +try: + pass +except* (RuntimeError, Bad) as e: # E: Exception type must be derived from BaseException + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[Any]" +[builtins fixtures/exception.pyi] + +[case testTryStarGroupInvalid] +try: + pass +except* ExceptionGroup as e: # E: Exception type in except* cannot derive from BaseExceptionGroup + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[Any]" +[builtins fixtures/exception.pyi] + +[case testTryStarGroupInvalidTuple] +try: + pass +except* (RuntimeError, ExceptionGroup) as e: # E: Exception type in except* cannot derive from BaseExceptionGroup + reveal_type(e) # N: Revealed type is "builtins.ExceptionGroup[Union[builtins.RuntimeError, Any]]" +[builtins fixtures/exception.pyi] diff --git a/test-data/unit/check-python38.test b/test-data/unit/check-python38.test index deded7a52f72..1922192c2877 100644 --- a/test-data/unit/check-python38.test +++ b/test-data/unit/check-python38.test @@ -115,21 +115,25 @@ def g(x: int): ... ) # type: ignore # E: Unused "type: ignore" comment [case testPEP570ArgTypesMissing] -# flags: --disallow-untyped-defs +# flags: --python-version 3.8 --disallow-untyped-defs def f(arg, /) -> None: ... # E: Function is missing a type annotation for one or more arguments [case testPEP570ArgTypesBadDefault] +# flags: --python-version 3.8 def f(arg: int = "ERROR", /) -> None: ... # E: Incompatible default for argument "arg" (default has type "str", argument has type "int") [case testPEP570ArgTypesDefault] +# flags: --python-version 3.8 def f(arg: int = 0, /) -> None: reveal_type(arg) # N: Revealed type is "builtins.int" [case testPEP570ArgTypesRequired] +# flags: --python-version 3.8 def f(arg: int, /) -> None: reveal_type(arg) # N: Revealed type is "builtins.int" [case testPEP570Required] +# flags: --python-version 3.8 def f(arg: int, /) -> None: ... # N: "f" defined here f(1) f("ERROR") # E: Argument 1 to "f" has incompatible type "str"; expected "int" @@ -137,6 +141,7 @@ f(arg=1) # E: Unexpected keyword argument "arg" for "f" f(arg="ERROR") # E: Unexpected keyword argument "arg" for "f" [case testPEP570Default] +# flags: --python-version 3.8 def f(arg: int = 0, /) -> None: ... # N: "f" defined here f() f(1) @@ -145,6 +150,7 @@ f(arg=1) # E: Unexpected keyword argument "arg" for "f" f(arg="ERROR") # E: Unexpected keyword argument "arg" for "f" [case testPEP570Calls] +# flags: --python-version 3.8 --no-strict-optional from typing import Any, Dict def f(p, /, p_or_kw, *, kw) -> None: ... # N: "f" defined here d = None # type: Dict[Any, Any] @@ -157,6 +163,7 @@ f(**d) # E: Missing positional argument "p_or_kw" in call to "f" [builtins fixtures/dict.pyi] [case testPEP570Signatures1] +# flags: --python-version 3.8 def f(p1: bytes, p2: float, /, p_or_kw: int, *, kw: str) -> None: reveal_type(p1) # N: Revealed type is "builtins.bytes" reveal_type(p2) # N: Revealed type is "builtins.float" @@ -164,6 +171,7 @@ def f(p1: bytes, p2: float, /, p_or_kw: int, *, kw: str) -> None: reveal_type(kw) # N: Revealed type is "builtins.str" [case testPEP570Signatures2] +# flags: --python-version 3.8 def f(p1: bytes, p2: float = 0.0, /, p_or_kw: int = 0, *, kw: str) -> None: reveal_type(p1) # N: Revealed type is "builtins.bytes" reveal_type(p2) # N: Revealed type is "builtins.float" @@ -171,28 +179,33 @@ def f(p1: bytes, p2: float = 0.0, /, p_or_kw: int = 0, *, kw: str) -> None: reveal_type(kw) # N: Revealed type is "builtins.str" [case testPEP570Signatures3] +# flags: --python-version 3.8 def f(p1: bytes, p2: float = 0.0, /, *, kw: int) -> None: reveal_type(p1) # N: Revealed type is "builtins.bytes" reveal_type(p2) # N: Revealed type is "builtins.float" reveal_type(kw) # N: Revealed type is "builtins.int" [case testPEP570Signatures4] +# flags: --python-version 3.8 def f(p1: bytes, p2: int = 0, /) -> None: reveal_type(p1) # N: Revealed type is "builtins.bytes" reveal_type(p2) # N: Revealed type is "builtins.int" [case testPEP570Signatures5] +# flags: --python-version 3.8 def f(p1: bytes, p2: float, /, p_or_kw: int) -> None: reveal_type(p1) # N: Revealed type is "builtins.bytes" reveal_type(p2) # N: Revealed type is "builtins.float" reveal_type(p_or_kw) # N: Revealed type is "builtins.int" [case testPEP570Signatures6] +# flags: --python-version 3.8 def f(p1: bytes, p2: float, /) -> None: reveal_type(p1) # N: Revealed type is "builtins.bytes" reveal_type(p2) # N: Revealed type is "builtins.float" [case testPEP570Unannotated] +# flags: --python-version 3.8 def f(arg, /): ... # N: "f" defined here g = lambda arg, /: arg def h(arg=0, /): ... # N: "h" defined here @@ -297,7 +310,7 @@ def f(x: int = (c := 4)) -> int: z2: NT # E: Variable "NT" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases - if Alias := int: + if Alias := int: # E: Function "Type[int]" could always be true in boolean context z3: Alias # E: Variable "Alias" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases @@ -386,7 +399,7 @@ reveal_type(z2) # E: Name "z2" is not defined # N: Revealed type is "Any" [case testWalrusConditionalTypeBinder] # flags: --python-version 3.8 -from typing import Union +from typing import Tuple, Union from typing_extensions import Literal class Good: @@ -403,7 +416,14 @@ if (thing := get_thing()).is_good: reveal_type(thing) # N: Revealed type is "__main__.Good" else: reveal_type(thing) # N: Revealed type is "__main__.Bad" -[builtins fixtures/property.pyi] + +def get_things() -> Union[Tuple[Good], Tuple[Bad]]: ... + +if (things := get_things())[0].is_good: + reveal_type(things) # N: Revealed type is "Tuple[__main__.Good]" +else: + reveal_type(things) # N: Revealed type is "Tuple[__main__.Bad]" +[builtins fixtures/list.pyi] [case testWalrusConditionalTypeCheck] # flags: --strict-optional --python-version 3.8 @@ -554,6 +574,7 @@ def foo() -> None: [builtins fixtures/dict.pyi] [case testOverloadWithPositionalOnlySelf] +# flags: --python-version 3.8 from typing import overload, Optional class Foo: @@ -576,3 +597,124 @@ class Bar: def f(self, a: Optional[str] = None, /, *, b: bool = False) -> None: ... [builtins fixtures/bool.pyi] + +[case testOverloadPositionalOnlyErrorMessage] +# flags: --python-version 3.8 +from typing import overload + +@overload +def foo(a: int, /): ... +@overload +def foo(a: str): ... +def foo(a): ... + +foo(a=1) +[out] +main:10: error: No overload variant of "foo" matches argument type "int" +main:10: note: Possible overload variants: +main:10: note: def foo(int, /) -> Any +main:10: note: def foo(a: str) -> Any + +[case testOverloadPositionalOnlyErrorMessageAllTypes] +# flags: --python-version 3.8 +from typing import overload + +@overload +def foo(a: int, /, b: int, *, c: int): ... +@overload +def foo(a: str, b: int, *, c: int): ... +def foo(a, b, *, c): ... + +foo(a=1) +[out] +main:10: error: No overload variant of "foo" matches argument type "int" +main:10: note: Possible overload variants: +main:10: note: def foo(int, /, b: int, *, c: int) -> Any +main:10: note: def foo(a: str, b: int, *, c: int) -> Any + +[case testOverloadPositionalOnlyErrorMessageMultiplePosArgs] +# flags: --python-version 3.8 +from typing import overload + +@overload +def foo(a: int, b: int, c: int, /, d: str): ... +@overload +def foo(a: str, b: int, c: int, d: str): ... +def foo(a, b, c, d): ... + +foo(a=1) +[out] +main:10: error: No overload variant of "foo" matches argument type "int" +main:10: note: Possible overload variants: +main:10: note: def foo(int, int, int, /, d: str) -> Any +main:10: note: def foo(a: str, b: int, c: int, d: str) -> Any + +[case testOverloadPositionalOnlyErrorMessageMethod] +# flags: --python-version 3.8 +from typing import overload + +class Some: + @overload + def foo(self, __a: int): ... + @overload + def foo(self, a: float, /): ... + @overload + def foo(self, a: str): ... + def foo(self, a): ... + +Some().foo(a=1) +[out] +main:13: error: No overload variant of "foo" of "Some" matches argument type "int" +main:13: note: Possible overload variants: +main:13: note: def foo(self, int, /) -> Any +main:13: note: def foo(self, float, /) -> Any +main:13: note: def foo(self, a: str) -> Any + +[case testOverloadPositionalOnlyErrorMessageClassMethod] +# flags: --python-version 3.8 +from typing import overload + +class Some: + @overload + @classmethod + def foo(cls, __a: int): ... + @overload + @classmethod + def foo(cls, a: float, /): ... + @overload + @classmethod + def foo(cls, a: str): ... + @classmethod + def foo(cls, a): ... + +Some.foo(a=1) +[builtins fixtures/classmethod.pyi] +[out] +main:17: error: No overload variant of "foo" of "Some" matches argument type "int" +main:17: note: Possible overload variants: +main:17: note: def foo(cls, int, /) -> Any +main:17: note: def foo(cls, float, /) -> Any +main:17: note: def foo(cls, a: str) -> Any + +[case testUnpackWithDuplicateNamePositionalOnly] +# flags: --python-version 3.8 +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int +def foo(name: str, /, **kwargs: Unpack[Person]) -> None: # Allowed + ... +[builtins fixtures/dict.pyi] + +[case testPartiallyDefinedWithAssignmentExpr] +# flags: --python-version 3.8 --enable-error-code partially-defined +def f1() -> None: + d = {0: 1} + if int(): + x = 1 + + if (x := d[x]) is None: # E: Name "x" may be undefined + y = x + z = x +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index c326246436ba..0d727b109658 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -1,7 +1,6 @@ -- Tests checking that basic functionality works [case testRecursiveAliasBasic] -# flags: --enable-recursive-aliases from typing import Dict, List, Union, TypeVar, Sequence JSON = Union[str, List[JSON], Dict[str, JSON]] @@ -17,7 +16,6 @@ x = ["foo", {"bar": [Bad()]}] # E: List item 0 has incompatible type "Bad"; exp [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasBasicGenericSubtype] -# flags: --enable-recursive-aliases from typing import Union, TypeVar, Sequence, List T = TypeVar("T") @@ -37,7 +35,6 @@ xx = yy # OK [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasBasicGenericInference] -# flags: --enable-recursive-aliases from typing import Union, TypeVar, Sequence, List T = TypeVar("T") @@ -61,7 +58,6 @@ x = [1, [Bad()]] # E: List item 0 has incompatible type "Bad"; expected "Union[ [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasGenericInferenceNested] -# flags: --enable-recursive-aliases from typing import Union, TypeVar, Sequence, List T = TypeVar("T") @@ -77,7 +73,6 @@ reveal_type(flatten([[B(), [[B()]]]])) # N: Revealed type is "builtins.list[__m [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasNewStyleSupported] -# flags: --enable-recursive-aliases from test import A x: A @@ -93,7 +88,6 @@ A = int | list[A] -- Tests duplicating some existing type alias tests with recursive aliases enabled [case testRecursiveAliasesMutual] -# flags: --enable-recursive-aliases from typing import Type, Callable, Union A = Union[B, int] @@ -103,7 +97,6 @@ x: A reveal_type(x) # N: Revealed type is "Union[def (Union[Type[def (...) -> builtins.int], Type[builtins.int]]) -> builtins.int, builtins.int]" [case testRecursiveAliasesProhibited-skip] -# flags: --enable-recursive-aliases from typing import Type, Callable, Union A = Union[B, int] @@ -111,7 +104,6 @@ B = Union[A, int] C = Type[C] [case testRecursiveAliasImported] -# flags: --enable-recursive-aliases import lib x: lib.A reveal_type(x) # N: Revealed type is "builtins.list[builtins.list[...]]" @@ -128,7 +120,6 @@ B = List[A] [builtins fixtures/list.pyi] [case testRecursiveAliasViaBaseClass] -# flags: --enable-recursive-aliases from typing import List x: B @@ -140,7 +131,6 @@ reveal_type(x[0][0]) # N: Revealed type is "__main__.C" [builtins fixtures/list.pyi] [case testRecursiveAliasViaBaseClass2] -# flags: --enable-recursive-aliases from typing import NewType, List x: D @@ -154,7 +144,6 @@ class B(D): [builtins fixtures/list.pyi] [case testRecursiveAliasViaBaseClass3] -# flags: --enable-recursive-aliases from typing import List, Generic, TypeVar, NamedTuple T = TypeVar('T') @@ -173,7 +162,6 @@ reveal_type(x) # N: Revealed type is "__main__.G[Tuple[builtins.int, fallback=_ [builtins fixtures/list.pyi] [case testRecursiveAliasViaBaseClassImported] -# flags: --enable-recursive-aliases import a [file a.py] from typing import List @@ -190,7 +178,6 @@ reveal_type(f) # N: Revealed type is "def (x: builtins.list[a.C]) -> builtins.l [builtins fixtures/list.pyi] [case testRecursiveAliasViaNamedTuple] -# flags: --enable-recursive-aliases from typing import List, NamedTuple, Union Exp = Union['A', 'B'] @@ -210,7 +197,6 @@ my_eval(A([B(1), B(2)])) [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasesSimplifiedUnion] -# flags: --enable-recursive-aliases from typing import Sequence, TypeVar, Union class A: ... @@ -231,7 +217,6 @@ x = y # E: Incompatible types in assignment (expression has type "Sequence[Unio [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasesJoins] -# flags: --enable-recursive-aliases from typing import Sequence, TypeVar, Union class A: ... @@ -257,7 +242,6 @@ x = y3 # E: Incompatible types in assignment (expression has type "Sequence[Uni [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasesRestrictions] -# flags: --enable-recursive-aliases from typing import Sequence, Mapping, Union A = Sequence[Union[int, A]] @@ -272,7 +256,6 @@ else: [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasesRestrictions2] -# flags: --enable-recursive-aliases from typing import Sequence, Union class A: ... @@ -296,7 +279,6 @@ if isinstance(b[0], Sequence): [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasWithRecursiveInstance] -# flags: --enable-recursive-aliases from typing import Sequence, Union, TypeVar class A: ... @@ -317,7 +299,6 @@ reveal_type(join(b, a)) # N: Revealed type is "typing.Sequence[Union[__main__.A [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasWithRecursiveInstanceInference] -# flags: --enable-recursive-aliases from typing import Sequence, Union, TypeVar, List T = TypeVar("T") @@ -338,7 +319,6 @@ reveal_type(bar(nib)) # N: Revealed type is "__main__.B" [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasTopUnion] -# flags: --enable-recursive-aliases from typing import Sequence, Union, TypeVar, List class A: ... @@ -363,7 +343,6 @@ reveal_type(foo(xx)) # N: Revealed type is "__main__.B" [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasInferenceExplicitNonRecursive] -# flags: --enable-recursive-aliases from typing import Sequence, Union, TypeVar, List T = TypeVar("T") @@ -390,7 +369,6 @@ reveal_type(bar(llla)) # N: Revealed type is "__main__.A" [builtins fixtures/isinstancelist.pyi] [case testRecursiveAliasesWithOptional] -# flags: --enable-recursive-aliases from typing import Optional, Sequence A = Sequence[Optional[A]] @@ -398,7 +376,6 @@ x: A y: str = x[0] # E: Incompatible types in assignment (expression has type "Optional[A]", variable has type "str") [case testRecursiveAliasesProhibitBadAliases] -# flags: --enable-recursive-aliases from typing import Union, Type, List, TypeVar NR = List[int] @@ -440,7 +417,7 @@ reveal_type(d) # N: Revealed type is "Any" [builtins fixtures/isinstancelist.pyi] [case testBasicRecursiveNamedTuple] -# flags: --enable-recursive-aliases +# flags: --strict-optional from typing import NamedTuple, Optional NT = NamedTuple("NT", [("x", Optional[NT]), ("y", int)]) @@ -454,7 +431,6 @@ if nt.x is not None: [builtins fixtures/tuple.pyi] [case testBasicRecursiveNamedTupleSpecial] -# flags: --enable-recursive-aliases from typing import NamedTuple, TypeVar, Tuple NT = NamedTuple("NT", [("x", NT), ("y", int)]) @@ -476,7 +452,7 @@ reveal_type(f(tnt, nt)) # N: Revealed type is "builtins.tuple[Any, ...]" [builtins fixtures/tuple.pyi] [case testBasicRecursiveNamedTupleClass] -# flags: --enable-recursive-aliases +# flags: --strict-optional from typing import NamedTuple, Optional class NT(NamedTuple): @@ -493,7 +469,6 @@ if nt.x is not None: [builtins fixtures/tuple.pyi] [case testRecursiveRegularTupleClass] -# flags: --enable-recursive-aliases from typing import Tuple x: B @@ -505,7 +480,6 @@ reveal_type(b.x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] [case testRecursiveTupleClassesNewType] -# flags: --enable-recursive-aliases from typing import Tuple, NamedTuple, NewType x: C @@ -528,7 +502,6 @@ reveal_type(bnt.y) # N: Revealed type is "builtins.int" -- Tests duplicating some existing named tuple tests with recursive aliases enabled [case testMutuallyRecursiveNamedTuples] -# flags: --enable-recursive-aliases from typing import Tuple, NamedTuple, TypeVar, Union A = NamedTuple('A', [('x', str), ('y', Tuple[B, ...])]) @@ -547,7 +520,6 @@ y: str = x # E: Incompatible types in assignment (expression has type "Union[st [builtins fixtures/tuple.pyi] [case testMutuallyRecursiveNamedTuplesJoin] -# flags: --enable-recursive-aliases from typing import NamedTuple, Tuple class B(NamedTuple): @@ -560,11 +532,17 @@ m: A s: str = n.x # E: Incompatible types in assignment (expression has type "Tuple[A, int]", variable has type "str") reveal_type(m[0]) # N: Revealed type is "builtins.str" lst = [m, n] -reveal_type(lst[0]) # N: Revealed type is "Tuple[builtins.object, builtins.object]" + +# Unfortunately, join of two recursive types is not very precise. +reveal_type(lst[0]) # N: Revealed type is "builtins.object" + +# These just should not crash +lst1 = [m] +lst2 = [m, m] +lst3 = [m, m, m] [builtins fixtures/tuple.pyi] [case testMutuallyRecursiveNamedTuplesClasses] -# flags: --enable-recursive-aliases from typing import NamedTuple, Tuple class B(NamedTuple): @@ -587,7 +565,6 @@ t = m # E: Incompatible types in assignment (expression has type "B", variable [builtins fixtures/tuple.pyi] [case testMutuallyRecursiveNamedTuplesCalls] -# flags: --enable-recursive-aliases from typing import NamedTuple B = NamedTuple('B', [('x', A), ('y', int)]) @@ -600,7 +577,6 @@ f(n) # E: Argument 1 to "f" has incompatible type "A"; expected "B" [builtins fixtures/tuple.pyi] [case testNoRecursiveTuplesAtFunctionScope] -# flags: --enable-recursive-aliases from typing import NamedTuple, Tuple def foo() -> None: class B(NamedTuple): @@ -608,11 +584,10 @@ def foo() -> None: # N: Recursive types are not allowed at function scope y: int b: B - reveal_type(b) # N: Revealed type is "Tuple[Any, builtins.int, fallback=__main__.B@4]" + reveal_type(b) # N: Revealed type is "Tuple[Any, builtins.int, fallback=__main__.B@3]" [builtins fixtures/tuple.pyi] [case testBasicRecursiveGenericNamedTuple] -# flags: --enable-recursive-aliases from typing import Generic, NamedTuple, TypeVar, Union T = TypeVar("T", covariant=True) @@ -636,7 +611,6 @@ reveal_type(last(ntb)) # N: Revealed type is "__main__.B" [builtins fixtures/tuple.pyi] [case testBasicRecursiveTypedDictClass] -# flags: --enable-recursive-aliases from typing import TypedDict class TD(TypedDict): @@ -650,7 +624,6 @@ s: str = td["y"] # E: Incompatible types in assignment (expression has type "TD [typing fixtures/typing-typeddict.pyi] [case testBasicRecursiveTypedDictCall] -# flags: --enable-recursive-aliases from typing import TypedDict TD = TypedDict("TD", {"x": int, "y": TD}) @@ -668,7 +641,6 @@ td = td3 # E: Incompatible types in assignment (expression has type "TD3", vari [typing fixtures/typing-typeddict.pyi] [case testBasicRecursiveTypedDictExtending] -# flags: --enable-recursive-aliases from typing import TypedDict class TDA(TypedDict): @@ -689,7 +661,6 @@ reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'xb': builtins [typing fixtures/typing-typeddict.pyi] [case testRecursiveTypedDictCreation] -# flags: --enable-recursive-aliases from typing import TypedDict, Optional class TD(TypedDict): @@ -705,7 +676,7 @@ itd2 = TD(x=0, y=TD(x=0, y=TD(x=0, y=None))) [typing fixtures/typing-typeddict.pyi] [case testRecursiveTypedDictMethods] -# flags: --enable-recursive-aliases +# flags: --strict-optional from typing import TypedDict class TD(TypedDict, total=False): @@ -725,7 +696,6 @@ td.update({"x": 0, "y": {"x": 1, "y": {"x": 2, "y": 42}}}) # E: Incompatible ty [typing fixtures/typing-typeddict.pyi] [case testRecursiveTypedDictSubtyping] -# flags: --enable-recursive-aliases from typing import TypedDict class TDA1(TypedDict): @@ -752,7 +722,6 @@ fb(tda1) # E: Argument 1 to "fb" has incompatible type "TDA1"; expected "TDB" [typing fixtures/typing-typeddict.pyi] [case testRecursiveTypedDictJoin] -# flags: --enable-recursive-aliases from typing import TypedDict, TypeVar class TDA1(TypedDict): @@ -778,7 +747,6 @@ reveal_type(f(tda1, tdb)) # N: Revealed type is "TypedDict({})" [typing fixtures/typing-typeddict.pyi] [case testBasicRecursiveGenericTypedDict] -# flags: --enable-recursive-aliases from typing import TypedDict, TypeVar, Generic, Optional, List T = TypeVar("T") @@ -794,7 +762,6 @@ reveal_type(collect({"left": {"right": {"value": 0}}})) # N: Revealed type is " [typing fixtures/typing-typeddict.pyi] [case testRecursiveGenericTypedDictExtending] -# flags: --enable-recursive-aliases from typing import TypedDict, Generic, TypeVar, List T = TypeVar("T") @@ -810,3 +777,52 @@ std: STD[str] reveal_type(std) # N: Revealed type is "TypedDict('__main__.STD', {'val': builtins.str, 'other': ..., 'sval': builtins.str, 'one': TypedDict('__main__.TD', {'val': builtins.str, 'other': ...})})" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testRecursiveClassLevelAlias] +# flags: --strict-optional +from typing import Union, Sequence + +class A: + Children = Union[Sequence['Children'], 'A', None] +x: A.Children +reveal_type(x) # N: Revealed type is "Union[typing.Sequence[...], __main__.A, None]" + +class B: + Foo = Sequence[Bar] + Bar = Sequence[Foo] +y: B.Foo +reveal_type(y) # N: Revealed type is "typing.Sequence[typing.Sequence[...]]" +[builtins fixtures/tuple.pyi] + +[case testNoCrashOnRecursiveTupleFallback] +from typing import Union, Tuple + +Tree1 = Union[str, Tuple[Tree1]] +Tree2 = Union[str, Tuple[Tree2, Tree2]] +Tree3 = Union[str, Tuple[Tree3, Tree3, Tree3]] + +def test1() -> Tree1: + return 42 # E: Incompatible return value type (got "int", expected "Union[str, Tuple[Tree1]]") +def test2() -> Tree2: + return 42 # E: Incompatible return value type (got "int", expected "Union[str, Tuple[Tree2, Tree2]]") +def test3() -> Tree3: + return 42 # E: Incompatible return value type (got "int", expected "Union[str, Tuple[Tree3, Tree3, Tree3]]") +[builtins fixtures/tuple.pyi] + +[case testRecursiveDoubleUnionNoCrash] +from typing import Tuple, Union, Callable, Sequence + +K = Union[int, Tuple[Union[int, K]]] +L = Union[int, Callable[[], Union[int, L]]] +M = Union[int, Sequence[Union[int, M]]] + +x: K +x = x +y: L +y = y +z: M +z = z + +x = y # E: Incompatible types in assignment (expression has type "L", variable has type "K") +z = x # OK +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 9768f43d0bb1..4be5060996e2 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -945,6 +945,18 @@ x = f() main:10: note: Revealed type is "builtins.int" main:15: note: Revealed type is "builtins.str" +[case testExceptionVariableWithDisallowAnyExprInDeferredNode] +# flags: --disallow-any-expr +def f() -> int: + x + try: + pass + except Exception as ex: + pass + return 0 +x = f() +[builtins fixtures/exception.pyi] + [case testArbitraryExpressionAsExceptionType] import typing a = BaseException @@ -1073,6 +1085,10 @@ a = A() del a.x, a.y # E: "A" has no attribute "y" [builtins fixtures/tuple.pyi] +[case testDelStmtWithTypeInfo] +class Foo: ... +del Foo +Foo + 1 # E: Trying to read deleted variable "Foo" [case testDelStatementWithAssignmentSimple] a = 1 @@ -2170,7 +2186,7 @@ N = TypedDict('N', {'x': int}) [out] [case testGlobalWithoutInitialization] - +# flags: --disable-error-code=annotation-unchecked from typing import List def foo() -> None: @@ -2184,3 +2200,9 @@ def foo2(): bar2 = [] # type: List[str] bar2 [builtins fixtures/list.pyi] + +[case testNoteUncheckedAnnotation] +def foo(): + x: int = "no" # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs + y = "no" # type: int # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs + z: int # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs diff --git a/test-data/unit/check-super.test b/test-data/unit/check-super.test index b9f6638d391a..0913f4f25126 100644 --- a/test-data/unit/check-super.test +++ b/test-data/unit/check-super.test @@ -380,3 +380,32 @@ class A: class B(A): def h(self, t: Type[None]) -> None: super(t, self).f # E: Unsupported argument 1 for "super" + +[case testSuperSelfTypeInstanceMethod] +from typing import TypeVar, Type + +T = TypeVar("T", bound="A") + +class A: + def foo(self: T) -> T: ... + +class B(A): + def foo(self: T) -> T: + reveal_type(super().foo()) # N: Revealed type is "T`-1" + return super().foo() + +[case testSuperSelfTypeClassMethod] +from typing import TypeVar, Type + +T = TypeVar("T", bound="A") + +class A: + @classmethod + def foo(cls: Type[T]) -> T: ... + +class B(A): + @classmethod + def foo(cls: Type[T]) -> T: + reveal_type(super().foo()) # N: Revealed type is "T`-1" + return super().foo() +[builtins fixtures/classmethod.pyi] diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index c6ae9e808f8a..061a4bcfa48d 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1228,8 +1228,8 @@ y = "" reveal_type(t[x]) # N: Revealed type is "Union[builtins.int, builtins.str]" t[y] # E: No overload variant of "__getitem__" of "tuple" matches argument type "str" \ # N: Possible overload variants: \ - # N: def __getitem__(self, int) -> object \ - # N: def __getitem__(self, slice) -> Tuple[object, ...] + # N: def __getitem__(self, int, /) -> object \ + # N: def __getitem__(self, slice, /) -> Tuple[object, ...] [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 95fe483ac116..8dafc8f47a6c 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -197,7 +197,7 @@ Alias = Tuple[int, T] [out] [case testRecursiveAliasesErrors1] - +# flags: --disable-recursive-aliases # Recursive aliases are not supported yet. from typing import Type, Callable, Union @@ -206,7 +206,7 @@ B = Callable[[B], int] # E: Cannot resolve name "B" (possible cyclic definition) C = Type[C] # E: Cannot resolve name "C" (possible cyclic definition) [case testRecursiveAliasesErrors2] - +# flags: --disable-recursive-aliases # Recursive aliases are not supported yet. from typing import Type, Callable, Union @@ -243,8 +243,7 @@ reveal_type(x[0].x) # N: Revealed type is "builtins.str" [out] [case testJSONAliasApproximation] - -# Recursive aliases are not supported yet. +# flags: --disable-recursive-aliases from typing import List, Union, Dict x: JSON # E: Cannot resolve name "JSON" (possible cyclic definition) JSON = Union[int, str, List[JSON], Dict[str, JSON]] # E: Cannot resolve name "JSON" (possible cyclic definition) @@ -772,7 +771,6 @@ f(string, string) [typing fixtures/typing-medium.pyi] [case testForwardTypeVarRefWithRecursiveFlag] -# flags: --enable-recursive-aliases import c [file a.py] from typing import TypeVar, List, Any, Generic @@ -796,3 +794,154 @@ S = TypeVar("S") class C(Generic[S], List[Defer]): ... class Defer: ... [builtins fixtures/list.pyi] + +[case testClassLevelTypeAliasesInUnusualContexts] +from typing import Union +from typing_extensions import TypeAlias + +class Foo: pass + +NormalImplicit = Foo +NormalExplicit: TypeAlias = Foo +SpecialImplicit = Union[int, str] +SpecialExplicit: TypeAlias = Union[int, str] + +class Parent: + NormalImplicit = Foo + NormalExplicit: TypeAlias = Foo + SpecialImplicit = Union[int, str] + SpecialExplicit: TypeAlias = Union[int, str] + +class Child(Parent): pass + +p = Parent() +c = Child() + +# Use type aliases in a runtime context + +reveal_type(NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(SpecialImplicit) # N: Revealed type is "builtins.object" +reveal_type(SpecialExplicit) # N: Revealed type is "builtins.object" + +reveal_type(Parent.NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(Parent.NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(Parent.SpecialImplicit) # N: Revealed type is "builtins.object" +reveal_type(Parent.SpecialExplicit) # N: Revealed type is "builtins.object" + +reveal_type(Child.NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(Child.NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(Child.SpecialImplicit) # N: Revealed type is "builtins.object" +reveal_type(Child.SpecialExplicit) # N: Revealed type is "builtins.object" + +reveal_type(p.NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(p.NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(p.SpecialImplicit) # N: Revealed type is "builtins.object" +reveal_type(p.SpecialExplicit) # N: Revealed type is "builtins.object" + +reveal_type(c.NormalImplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(p.NormalExplicit) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(c.SpecialImplicit) # N: Revealed type is "builtins.object" +reveal_type(c.SpecialExplicit) # N: Revealed type is "builtins.object" + +# Use type aliases in a type alias context in a plausible way + +def plausible_top_1() -> NormalImplicit: pass +def plausible_top_2() -> NormalExplicit: pass +def plausible_top_3() -> SpecialImplicit: pass +def plausible_top_4() -> SpecialExplicit: pass +reveal_type(plausible_top_1) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(plausible_top_2) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(plausible_top_3) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" +reveal_type(plausible_top_4) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" + +def plausible_parent_1() -> Parent.NormalImplicit: pass # E: Variable "__main__.Parent.NormalImplicit" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +def plausible_parent_2() -> Parent.NormalExplicit: pass +def plausible_parent_3() -> Parent.SpecialImplicit: pass +def plausible_parent_4() -> Parent.SpecialExplicit: pass +reveal_type(plausible_parent_1) # N: Revealed type is "def () -> Parent.NormalImplicit?" +reveal_type(plausible_parent_2) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(plausible_parent_3) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" +reveal_type(plausible_parent_4) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" + +def plausible_child_1() -> Child.NormalImplicit: pass # E: Variable "__main__.Parent.NormalImplicit" is not valid as a type \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +def plausible_child_2() -> Child.NormalExplicit: pass +def plausible_child_3() -> Child.SpecialImplicit: pass +def plausible_child_4() -> Child.SpecialExplicit: pass +reveal_type(plausible_child_1) # N: Revealed type is "def () -> Child.NormalImplicit?" +reveal_type(plausible_child_2) # N: Revealed type is "def () -> __main__.Foo" +reveal_type(plausible_child_3) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" +reveal_type(plausible_child_4) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" + +# Use type aliases in a type alias context in an implausible way + +def weird_parent_1() -> p.NormalImplicit: pass # E: Name "p.NormalImplicit" is not defined +def weird_parent_2() -> p.NormalExplicit: pass # E: Name "p.NormalExplicit" is not defined +def weird_parent_3() -> p.SpecialImplicit: pass # E: Name "p.SpecialImplicit" is not defined +def weird_parent_4() -> p.SpecialExplicit: pass # E: Name "p.SpecialExplicit" is not defined +reveal_type(weird_parent_1) # N: Revealed type is "def () -> Any" +reveal_type(weird_parent_2) # N: Revealed type is "def () -> Any" +reveal_type(weird_parent_3) # N: Revealed type is "def () -> Any" +reveal_type(weird_parent_4) # N: Revealed type is "def () -> Any" + +def weird_child_1() -> c.NormalImplicit: pass # E: Name "c.NormalImplicit" is not defined +def weird_child_2() -> c.NormalExplicit: pass # E: Name "c.NormalExplicit" is not defined +def weird_child_3() -> c.SpecialImplicit: pass # E: Name "c.SpecialImplicit" is not defined +def weird_child_4() -> c.SpecialExplicit: pass # E: Name "c.SpecialExplicit" is not defined +reveal_type(weird_child_1) # N: Revealed type is "def () -> Any" +reveal_type(weird_child_2) # N: Revealed type is "def () -> Any" +reveal_type(weird_child_3) # N: Revealed type is "def () -> Any" +reveal_type(weird_child_4) # N: Revealed type is "def () -> Any" +[builtins fixtures/tuple.pyi] + +[case testMalformedTypeAliasRuntimeReassignments] +from typing import Union +from typing_extensions import TypeAlias + +class Foo: pass + +NormalImplicit = Foo +NormalExplicit: TypeAlias = Foo +SpecialImplicit = Union[int, str] +SpecialExplicit: TypeAlias = Union[int, str] + +class Parent: + NormalImplicit = Foo + NormalExplicit: TypeAlias = Foo + SpecialImplicit = Union[int, str] + SpecialExplicit: TypeAlias = Union[int, str] + +class Child(Parent): pass + +p = Parent() +c = Child() + +NormalImplicit = 4 # E: Cannot assign multiple types to name "NormalImplicit" without an explicit "Type[...]" annotation \ + # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +NormalExplicit = 4 # E: Cannot assign multiple types to name "NormalExplicit" without an explicit "Type[...]" annotation \ + # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +SpecialImplicit = 4 # E: Cannot assign multiple types to name "SpecialImplicit" without an explicit "Type[...]" annotation +SpecialExplicit = 4 # E: Cannot assign multiple types to name "SpecialExplicit" without an explicit "Type[...]" annotation + +Parent.NormalImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +Parent.NormalExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +Parent.SpecialImplicit = 4 +Parent.SpecialExplicit = 4 + +Child.NormalImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +Child.NormalExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +Child.SpecialImplicit = 4 +Child.SpecialExplicit = 4 + +p.NormalImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +p.NormalExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +p.SpecialImplicit = 4 +p.SpecialExplicit = 4 + +c.NormalImplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +c.NormalExplicit = 4 # E: Incompatible types in assignment (expression has type "int", variable has type "Type[Foo]") +c.SpecialImplicit = 4 +c.SpecialExplicit = 4 +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 49c1fe1c9279..4c68b7b692ff 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -478,9 +478,9 @@ fun2(a) # Error main:17: error: Argument 1 to "fun2" has incompatible type "A"; expected "StrIntMap" main:17: note: Following member(s) of "A" have conflicts: main:17: note: Expected: -main:17: note: def __getitem__(self, str) -> int +main:17: note: def __getitem__(self, str, /) -> int main:17: note: Got: -main:17: note: def __getitem__(self, str) -> object +main:17: note: def __getitem__(self, str, /) -> object [case testTypedDictWithSimpleProtocolInference] from typing_extensions import Protocol, TypedDict @@ -1443,7 +1443,7 @@ reveal_type(x['a']['b']) # N: Revealed type is "builtins.int" [case testSelfRecursiveTypedDictInheriting] from mypy_extensions import TypedDict - +# flags: --disable-recursive-aliases class MovieBase(TypedDict): name: str year: int @@ -1457,7 +1457,7 @@ reveal_type(m['director']['name']) # N: Revealed type is "Any" [out] [case testSubclassOfRecursiveTypedDict] - +# flags: --disable-recursive-aliases from typing import List from mypy_extensions import TypedDict @@ -2012,16 +2012,35 @@ v = {bad2: 2} # E: Extra key "bad" for TypedDict "Value" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] -[case testCannotUseFinalDecoratorWithTypedDict] +[case testCannotSubclassFinalTypedDict] from typing import TypedDict from typing_extensions import final -@final # E: @final cannot be used with TypedDict +@final class DummyTypedDict(TypedDict): int_val: int float_val: float str_val: str +class SubType(DummyTypedDict): # E: Cannot inherit from final class "DummyTypedDict" + pass + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testCannotSubclassFinalTypedDictWithForwardDeclarations] +from typing import TypedDict +from typing_extensions import final + +@final +class DummyTypedDict(TypedDict): + forward_declared: "ForwardDeclared" + +class SubType(DummyTypedDict): # E: Cannot inherit from final class "DummyTypedDict" + pass + +class ForwardDeclared: pass + [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] @@ -2526,12 +2545,28 @@ Alias(key=0, value=0) # E: Missing type parameters for generic type "Alias" \ [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] +[case testGenericTypedDictMultipleGenerics] +# See https://github.com/python/mypy/issues/13755 +from typing import Generic, TypeVar, TypedDict + +T = TypeVar("T") +Foo = TypedDict("Foo", {"bar": T}) +class Stack(Generic[T]): pass + +a = Foo[str] +b = Foo[int] +reveal_type(a) # N: Revealed type is "def (*, bar: builtins.str) -> TypedDict('__main__.Foo', {'bar': builtins.str})" +reveal_type(b) # N: Revealed type is "def (*, bar: builtins.int) -> TypedDict('__main__.Foo', {'bar': builtins.int})" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + [case testGenericTypedDictCallSyntax] from typing import TypedDict, TypeVar T = TypeVar("T") TD = TypedDict("TD", {"key": int, "value": T}) -reveal_type(TD) # N: Revealed type is "def [T] (*, key: builtins.int, value: T`-1) -> TypedDict('__main__.TD', {'key': builtins.int, 'value': T`-1})" +reveal_type(TD) # N: Revealed type is "def [T] (*, key: builtins.int, value: T`1) -> TypedDict('__main__.TD', {'key': builtins.int, 'value': T`1})" tds: TD[str] reveal_type(tds) # N: Revealed type is "TypedDict('__main__.TD', {'key': builtins.int, 'value': builtins.str})" diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 193d1b0a58ba..d8f6cde10441 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -95,19 +95,19 @@ reveal_type(h(args)) # N: Revealed type is "Tuple[builtins.str, builtins.str, b [builtins fixtures/tuple.pyi] [case testTypeVarTupleGenericClassDefn] -from typing import Generic, TypeVar, Tuple -from typing_extensions import TypeVarTuple +from typing import Generic, TypeVar, Tuple, Union +from typing_extensions import TypeVarTuple, Unpack T = TypeVar("T") Ts = TypeVarTuple("Ts") -class Variadic(Generic[Ts]): +class Variadic(Generic[Unpack[Ts]]): pass -class Mixed1(Generic[T, Ts]): +class Mixed1(Generic[T, Unpack[Ts]]): pass -class Mixed2(Generic[Ts, T]): +class Mixed2(Generic[Unpack[Ts], T]): pass variadic: Variadic[int, str] @@ -120,6 +120,13 @@ empty: Variadic[()] # TODO: fix pretty printer to be better. reveal_type(empty) # N: Revealed type is "__main__.Variadic" +bad: Variadic[Unpack[Tuple[int, ...]], str, Unpack[Tuple[bool, ...]]] # E: More than one Unpack in a type is not allowed +reveal_type(bad) # N: Revealed type is "__main__.Variadic[Unpack[builtins.tuple[builtins.int, ...]], builtins.str]" + +# TODO: This is tricky to fix because we need typeanal to know whether the current +# location is valid for an Unpack or not. +# bad2: Unpack[Tuple[int, ...]] + m1: Mixed1[int, str, bool] reveal_type(m1) # N: Revealed type is "__main__.Mixed1[builtins.int, builtins.str, builtins.bool]" @@ -133,7 +140,7 @@ Ts = TypeVarTuple("Ts") T = TypeVar("T") S = TypeVar("S") -class Variadic(Generic[T, Ts, S]): +class Variadic(Generic[T, Unpack[Ts], S]): pass def foo(t: Variadic[int, Unpack[Ts], object]) -> Tuple[int, Unpack[Ts]]: @@ -152,7 +159,7 @@ Ts = TypeVarTuple("Ts") T = TypeVar("T") S = TypeVar("S") -class Variadic(Generic[T, Ts, S]): +class Variadic(Generic[T, Unpack[Ts], S]): def __init__(self, t: Tuple[Unpack[Ts]]) -> None: ... @@ -170,3 +177,208 @@ from typing_extensions import TypeVarTuple Ts = TypeVarTuple("Ts") B = Ts # E: Type variable "__main__.Ts" is invalid as target for type alias [builtins fixtures/tuple.pyi] + +[case testPep646ArrayExample] +from typing import Generic, Tuple, TypeVar, Protocol, NewType +from typing_extensions import TypeVarTuple, Unpack + +Shape = TypeVarTuple('Shape') + +Height = NewType('Height', int) +Width = NewType('Width', int) + +T_co = TypeVar("T_co", covariant=True) +T = TypeVar("T") + +class SupportsAbs(Protocol[T_co]): + def __abs__(self) -> T_co: pass + +def abs(a: SupportsAbs[T]) -> T: + ... + +class Array(Generic[Unpack[Shape]]): + def __init__(self, shape: Tuple[Unpack[Shape]]): + self._shape: Tuple[Unpack[Shape]] = shape + + def get_shape(self) -> Tuple[Unpack[Shape]]: + return self._shape + + def __abs__(self) -> Array[Unpack[Shape]]: ... + + def __add__(self, other: Array[Unpack[Shape]]) -> Array[Unpack[Shape]]: ... + +shape = (Height(480), Width(640)) +x: Array[Height, Width] = Array(shape) +reveal_type(abs(x)) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" +reveal_type(x + x) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" + +[builtins fixtures/tuple.pyi] +[case testPep646ArrayExampleWithDType] +from typing import Generic, Tuple, TypeVar, Protocol, NewType +from typing_extensions import TypeVarTuple, Unpack + +DType = TypeVar("DType") +Shape = TypeVarTuple('Shape') + +Height = NewType('Height', int) +Width = NewType('Width', int) + +T_co = TypeVar("T_co", covariant=True) +T = TypeVar("T") + +class SupportsAbs(Protocol[T_co]): + def __abs__(self) -> T_co: pass + +def abs(a: SupportsAbs[T]) -> T: + ... + +class Array(Generic[DType, Unpack[Shape]]): + def __init__(self, shape: Tuple[Unpack[Shape]]): + self._shape: Tuple[Unpack[Shape]] = shape + + def get_shape(self) -> Tuple[Unpack[Shape]]: + return self._shape + + def __abs__(self) -> Array[DType, Unpack[Shape]]: ... + + def __add__(self, other: Array[DType, Unpack[Shape]]) -> Array[DType, Unpack[Shape]]: ... + +shape = (Height(480), Width(640)) +x: Array[float, Height, Width] = Array(shape) +reveal_type(abs(x)) # N: Revealed type is "__main__.Array[builtins.float, __main__.Height, __main__.Width]" +reveal_type(x + x) # N: Revealed type is "__main__.Array[builtins.float, __main__.Height, __main__.Width]" + +[builtins fixtures/tuple.pyi] + +[case testPep646ArrayExampleInfer] +from typing import Generic, Tuple, TypeVar, NewType +from typing_extensions import TypeVarTuple, Unpack + +Shape = TypeVarTuple('Shape') + +Height = NewType('Height', int) +Width = NewType('Width', int) + +class Array(Generic[Unpack[Shape]]): + pass + +x: Array[float, Height, Width] = Array() +[builtins fixtures/tuple.pyi] + +[case testPep646TypeConcatenation] +from typing import Generic, TypeVar, NewType +from typing_extensions import TypeVarTuple, Unpack + +Shape = TypeVarTuple('Shape') + +Channels = NewType("Channels", int) +Batch = NewType("Batch", int) +Height = NewType('Height', int) +Width = NewType('Width', int) + +class Array(Generic[Unpack[Shape]]): + pass + + +def add_batch_axis(x: Array[Unpack[Shape]]) -> Array[Batch, Unpack[Shape]]: ... +def del_batch_axis(x: Array[Batch, Unpack[Shape]]) -> Array[Unpack[Shape]]: ... +def add_batch_channels( + x: Array[Unpack[Shape]] +) -> Array[Batch, Unpack[Shape], Channels]: ... + +a: Array[Height, Width] +b = add_batch_axis(a) +reveal_type(b) # N: Revealed type is "__main__.Array[__main__.Batch, __main__.Height, __main__.Width]" +c = del_batch_axis(b) +reveal_type(c) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" +d = add_batch_channels(a) +reveal_type(d) # N: Revealed type is "__main__.Array[__main__.Batch, __main__.Height, __main__.Width, __main__.Channels]" + +[builtins fixtures/tuple.pyi] +[case testPep646TypeVarConcatenation] +from typing import Generic, TypeVar, NewType, Tuple +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar('T') +Ts = TypeVarTuple('Ts') + +def prefix_tuple( + x: T, + y: Tuple[Unpack[Ts]], +) -> Tuple[T, Unpack[Ts]]: + ... + +z = prefix_tuple(x=0, y=(True, 'a')) +reveal_type(z) # N: Revealed type is "Tuple[builtins.int, builtins.bool, builtins.str]" +[builtins fixtures/tuple.pyi] +[case testPep646TypeVarTupleUnpacking] +from typing import Generic, TypeVar, NewType, Any, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Shape = TypeVarTuple('Shape') + +Channels = NewType("Channels", int) +Batch = NewType("Batch", int) +Height = NewType('Height', int) +Width = NewType('Width', int) + +class Array(Generic[Unpack[Shape]]): + pass + +def process_batch_channels( + x: Array[Batch, Unpack[Tuple[Any, ...]], Channels] +) -> None: + ... + +x: Array[Batch, Height, Width, Channels] +process_batch_channels(x) +y: Array[Batch, Channels] +process_batch_channels(y) +z: Array[Batch] +process_batch_channels(z) # E: Argument 1 to "process_batch_channels" has incompatible type "Array[Batch]"; expected "Array[Batch, Unpack[Tuple[Any, ...]], Channels]" + +u: Array[Unpack[Tuple[Any, ...]]] + +def expect_variadic_array( + x: Array[Batch, Unpack[Shape]] +) -> None: + ... + +def expect_variadic_array_2( + x: Array[Batch, Height, Width, Channels] +) -> None: + ... + +expect_variadic_array(u) +expect_variadic_array_2(u) + +Ts = TypeVarTuple("Ts") +Ts2 = TypeVarTuple("Ts2") + +def bad(x: Tuple[int, Unpack[Ts], str, Unpack[Ts2]]) -> None: # E: More than one Unpack in a type is not allowed + + ... +reveal_type(bad) # N: Revealed type is "def [Ts, Ts2] (x: Tuple[builtins.int, Unpack[Ts`-1], builtins.str])" + +def bad2(x: Tuple[int, Unpack[Tuple[int, ...]], str, Unpack[Tuple[str, ...]]]) -> None: # E: More than one Unpack in a type is not allowed + ... +reveal_type(bad2) # N: Revealed type is "def (x: Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.str])" + + +[builtins fixtures/tuple.pyi] + +[case testPep646TypeVarStarArgs] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +# TODO: add less trivial tests with prefix/suffix etc. +# TODO: add tests that call with a type var tuple instead of just args. +def args_to_tuple(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: + reveal_type(args) # N: Revealed type is "Tuple[Unpack[Ts`-1]]" + return args + +reveal_type(args_to_tuple(1, 'a')) # N: Revealed type is "Tuple[Literal[1]?, Literal['a']?]" + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typevar-unbound.test b/test-data/unit/check-typevar-unbound.test index a233a9c7af13..d3e54c75e373 100644 --- a/test-data/unit/check-typevar-unbound.test +++ b/test-data/unit/check-typevar-unbound.test @@ -1,14 +1,23 @@ - [case testUnboundTypeVar] from typing import TypeVar T = TypeVar('T') -def f() -> T: # E: A function returning TypeVar should receive at least one argument containing the same Typevar +def f() -> T: # E: A function returning TypeVar should receive at least one argument containing the same TypeVar ... - f() +U = TypeVar('U', bound=int) + +def g() -> U: # E: A function returning TypeVar should receive at least one argument containing the same TypeVar \ + # N: Consider using the upper bound "int" instead + ... + +V = TypeVar('V', int, str) + +# TODO: this should also give an error +def h() -> V: + ... [case testInnerFunctionTypeVar] @@ -21,7 +30,6 @@ def g(a: T) -> T: ... return f() - [case testUnboundIterableOfTypeVars] from typing import Iterable, TypeVar @@ -29,7 +37,6 @@ T = TypeVar('T') def f() -> Iterable[T]: ... - f() [case testBoundTypeVar] @@ -40,7 +47,6 @@ T = TypeVar('T') def f(a: T, b: T, c: int) -> T: ... - [case testNestedBoundTypeVar] from typing import Callable, List, Union, Tuple, TypeVar @@ -58,3 +64,9 @@ def h(a: List[Union[Callable[..., T]]]) -> T: def j(a: List[Union[Callable[..., Tuple[T, T]], int]]) -> T: ... [builtins fixtures/tuple.pyi] + +[case testUnboundedTypevarUnpacking] +from typing import TypeVar +T = TypeVar("T") +def f(t: T) -> None: + a, *b = t # E: "object" object is not iterable diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index e772b489a6d2..a561c29e54f7 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -193,11 +193,19 @@ elif foo(): elif foo(): def f(x: Union[int, str, int, int, str]) -> None: pass elif foo(): - def f(x: Union[int, str, float]) -> None: pass # E: All conditional function variants must have identical signatures + def f(x: Union[int, str, float]) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: Union[int, str]) -> None \ + # N: Redefinition: \ + # N: def f(x: Union[int, str, float]) -> None elif foo(): def f(x: Union[S, T]) -> None: pass elif foo(): - def f(x: Union[str]) -> None: pass # E: All conditional function variants must have identical signatures + def f(x: Union[str]) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: Union[int, str]) -> None \ + # N: Redefinition: \ + # N: def f(x: str) -> None else: def f(x: Union[Union[int, T], Union[S, T], str]) -> None: pass @@ -206,7 +214,11 @@ else: if foo(): def g(x: Union[int, str, bytes]) -> None: pass else: - def g(x: Union[int, str]) -> None: pass # E: All conditional function variants must have identical signatures + def g(x: Union[int, str]) -> None: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def g(x: Union[int, str, bytes]) -> None \ + # N: Redefinition: \ + # N: def g(x: Union[int, str]) -> None [case testUnionSimplificationSpecialCases] from typing import Any, TypeVar, Union @@ -343,12 +355,12 @@ def foo(a: Union[A, B, C]): from typing import TypeVar, Union T = TypeVar('T') S = TypeVar('S') -def u(x: T, y: S) -> Union[S, T]: pass +def u(x: T, y: S) -> Union[T, S]: pass -reveal_type(u(1, 2.3)) # N: Revealed type is "builtins.float" -reveal_type(u(2.3, 1)) # N: Revealed type is "builtins.float" -reveal_type(u(False, 2.2)) # N: Revealed type is "builtins.float" -reveal_type(u(2.2, False)) # N: Revealed type is "builtins.float" +reveal_type(u(1, 2.3)) # N: Revealed type is "Union[builtins.int, builtins.float]" +reveal_type(u(2.3, 1)) # N: Revealed type is "Union[builtins.float, builtins.int]" +reveal_type(u(False, 2.2)) # N: Revealed type is "Union[builtins.bool, builtins.float]" +reveal_type(u(2.2, False)) # N: Revealed type is "Union[builtins.float, builtins.bool]" [builtins fixtures/primitives.pyi] [case testSimplifyingUnionWithTypeTypes1] @@ -479,7 +491,7 @@ class E: [case testUnionSimplificationWithBoolIntAndFloat] from typing import List, Union l = reveal_type([]) # type: List[Union[bool, int, float]] \ - # N: Revealed type is "builtins.list[builtins.float]" + # N: Revealed type is "builtins.list[Union[builtins.int, builtins.float]]" reveal_type(l) \ # N: Revealed type is "builtins.list[Union[builtins.bool, builtins.int, builtins.float]]" [builtins fixtures/list.pyi] @@ -487,7 +499,7 @@ reveal_type(l) \ [case testUnionSimplificationWithBoolIntAndFloat2] from typing import List, Union l = reveal_type([]) # type: List[Union[bool, int, float, str]] \ - # N: Revealed type is "builtins.list[Union[builtins.float, builtins.str]]" + # N: Revealed type is "builtins.list[Union[builtins.int, builtins.float, builtins.str]]" reveal_type(l) \ # N: Revealed type is "builtins.list[Union[builtins.bool, builtins.int, builtins.float, builtins.str]]" [builtins fixtures/list.pyi] @@ -533,7 +545,7 @@ from typing import Union, Tuple, Any a: Union[Tuple[int], Tuple[float]] (a1,) = a -reveal_type(a1) # N: Revealed type is "builtins.float" +reveal_type(a1) # N: Revealed type is "Union[builtins.int, builtins.float]" b: Union[Tuple[int], Tuple[str]] (b1,) = b @@ -546,7 +558,7 @@ from typing import Union, Tuple c: Union[Tuple[int, int], Tuple[int, float]] (c1, c2) = c reveal_type(c1) # N: Revealed type is "builtins.int" -reveal_type(c2) # N: Revealed type is "builtins.float" +reveal_type(c2) # N: Revealed type is "Union[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] [case testUnionMultiassignGeneric] @@ -613,7 +625,7 @@ b: Union[Tuple[float, int], Tuple[int, int]] b1: object b2: int (b1, b2) = b -reveal_type(b1) # N: Revealed type is "builtins.float" +reveal_type(b1) # N: Revealed type is "Union[builtins.float, builtins.int]" reveal_type(b2) # N: Revealed type is "builtins.int" c: Union[Tuple[int, int], Tuple[int, int]] @@ -627,7 +639,7 @@ d: Union[Tuple[int, int], Tuple[int, float]] d1: object (d1, d2) = d reveal_type(d1) # N: Revealed type is "builtins.int" -reveal_type(d2) # N: Revealed type is "builtins.float" +reveal_type(d2) # N: Revealed type is "Union[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] [case testUnionMultiassignIndexed] @@ -992,7 +1004,7 @@ def takes_int(arg: int) -> None: pass takes_int(x) # E: Argument 1 to "takes_int" has incompatible type "Union[ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[int], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[object], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[float], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[str], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[Any], ExtremelyLongTypeNameWhichIsGenericSoWeCanUseItMultipleTimes[bytes]]"; expected "int" [case testRecursiveForwardReferenceInUnion] - +# flags: --disable-recursive-aliases from typing import List, Union MYTYPE = List[Union[str, "MYTYPE"]] # E: Cannot resolve name "MYTYPE" (possible cyclic definition) [builtins fixtures/list.pyi] diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index 289d042d8790..48459dd8941a 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -242,7 +242,11 @@ import sys if sys.version_info >= (3, 5, 0): def foo() -> int: return 0 else: - def foo() -> str: return '' # E: All conditional function variants must have identical signatures + def foo() -> str: return '' # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def foo() -> int \ + # N: Redefinition: \ + # N: def foo() -> str [builtins fixtures/ops.pyi] [out] @@ -253,7 +257,11 @@ import sys if sys.version_info[1:] >= (5, 0): def foo() -> int: return 0 else: - def foo() -> str: return '' # E: All conditional function variants must have identical signatures + def foo() -> str: return '' # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def foo() -> int \ + # N: Redefinition: \ + # N: def foo() -> str [builtins fixtures/ops.pyi] [out] @@ -928,7 +936,8 @@ class Case1: return False and self.missing() # E: Right operand of "and" is never evaluated def test2(self) -> bool: - return not self.property_decorator_missing and self.missing() # E: Right operand of "and" is never evaluated + return not self.property_decorator_missing and self.missing() # E: Function "Callable[[], bool]" could always be true in boolean context \ + # E: Right operand of "and" is never evaluated def property_decorator_missing(self) -> bool: return True @@ -1397,3 +1406,20 @@ a or a # E: Right operand of "or" is never evaluated 1 and a and 1 # E: Right operand of "and" is never evaluated a and a # E: Right operand of "and" is never evaluated [builtins fixtures/exception.pyi] + +[case testUnreachableFlagWithTerminalBranchInDeferredNode] +# flags: --warn-unreachable +from typing import NoReturn + +def assert_never(x: NoReturn) -> NoReturn: ... + +def force_forward_ref() -> int: + return 4 + +def f(value: None) -> None: + x + if value is not None: + assert_never(value) + +x = force_forward_ref() +[builtins fixtures/exception.pyi] diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 4dc10c9f7489..00ac7df320d2 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -82,6 +82,7 @@ class C: pass [builtins fixtures/list.pyi] [case testCallingVarArgsFunctionWithDefaultArgs] +# flags: --implicit-optional --no-strict-optional a = None # type: A b = None # type: B @@ -388,12 +389,14 @@ class B(A): pass [builtins fixtures/list.pyi] [case testCallerVarArgsAndDefaultArgs] +# flags: --implicit-optional --no-strict-optional a, b = None, None # type: (A, B) -f(*()) # Fail -f(a, *[a]) # Fail -f(a, b, *[a]) # Fail -f(*(a, a, b)) # Fail +f(*()) # E: Too few arguments for "f" +f(a, *[a]) # E: Argument 2 to "f" has incompatible type "*List[A]"; expected "Optional[B]" \ + # E: Argument 2 to "f" has incompatible type "*List[A]"; expected "B" +f(a, b, *[a]) # E: Argument 3 to "f" has incompatible type "*List[A]"; expected "B" +f(*(a, a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, A, B]"; expected "Optional[B]" f(*(a,)) f(*(a, b)) f(*(a, b, b, b)) @@ -407,12 +410,6 @@ def f(a: 'A', b: 'B' = None, *c: 'B') -> None: class A: pass class B: pass [builtins fixtures/list.pyi] -[out] -main:3: error: Too few arguments for "f" -main:4: error: Argument 2 to "f" has incompatible type "*List[A]"; expected "Optional[B]" -main:4: error: Argument 2 to "f" has incompatible type "*List[A]"; expected "B" -main:5: error: Argument 3 to "f" has incompatible type "*List[A]"; expected "B" -main:6: error: Argument 1 to "f" has incompatible type "*Tuple[A, A, B]"; expected "Optional[B]" [case testVarArgsAfterKeywordArgInCall1] # see: mypy issue #2729 @@ -760,3 +757,327 @@ bar(*good3) bar(*bad1) # E: Argument 1 to "bar" has incompatible type "*I[str]"; expected "float" bar(*bad2) # E: List or tuple expected as variadic arguments [builtins fixtures/dict.pyi] + +-- Keyword arguments unpacking + +[case testUnpackKwargsReveal] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int +def foo(arg: bool, **kwargs: Unpack[Person]) -> None: ... + +reveal_type(foo) # N: Revealed type is "def (arg: builtins.bool, **kwargs: Unpack[TypedDict('__main__.Person', {'name': builtins.str, 'age': builtins.int})])" +[builtins fixtures/dict.pyi] + +[case testUnpackOutsideOfKwargs] +from typing_extensions import Unpack, TypedDict +class Person(TypedDict): + name: str + age: int + +def foo(x: Unpack[Person]) -> None: # E: TypedDict('__main__.Person', {'name': builtins.str, 'age': builtins.int}) cannot be unpacked (must be tuple or TypeVarTuple) + ... +def bar(x: int, *args: Unpack[Person]) -> None: # E: TypedDict('__main__.Person', {'name': builtins.str, 'age': builtins.int}) cannot be unpacked (must be tuple or TypeVarTuple) + ... +def baz(**kwargs: Unpack[Person]) -> None: # OK + ... +[builtins fixtures/dict.pyi] + +[case testUnpackWithoutTypedDict] +from typing_extensions import Unpack + +def foo(**kwargs: Unpack[dict]) -> None: # E: Unpack item in ** argument must be a TypedDict + ... +[builtins fixtures/dict.pyi] + +[case testUnpackWithDuplicateKeywords] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int +def foo(name: str, **kwargs: Unpack[Person]) -> None: # E: Overlap between argument names and ** TypedDict items: "name" + ... +[builtins fixtures/dict.pyi] + +[case testUnpackWithDuplicateKeywordKwargs] +from typing_extensions import Unpack, TypedDict +from typing import Dict, List + +class Spec(TypedDict): + args: List[int] + kwargs: Dict[int, int] +def foo(**kwargs: Unpack[Spec]) -> None: # Allowed + ... +foo(args=[1], kwargs={"2": 3}) # E: Dict entry 0 has incompatible type "str": "int"; expected "int": "int" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsNonIdentifier] +from typing_extensions import Unpack, TypedDict + +Weird = TypedDict("Weird", {"@": int}) + +def foo(**kwargs: Unpack[Weird]) -> None: + reveal_type(kwargs["@"]) # N: Revealed type is "builtins.int" +foo(**{"@": 42}) +foo(**{"no": "way"}) # E: Argument 1 to "foo" has incompatible type "**Dict[str, str]"; expected "int" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsEmpty] +from typing_extensions import Unpack, TypedDict + +Empty = TypedDict("Empty", {}) + +def foo(**kwargs: Unpack[Empty]) -> None: # N: "foo" defined here + reveal_type(kwargs) # N: Revealed type is "TypedDict('__main__.Empty', {})" +foo() +foo(x=1) # E: Unexpected keyword argument "x" for "foo" +[builtins fixtures/dict.pyi] + +[case testUnpackTypedDictTotality] +from typing_extensions import Unpack, TypedDict + +class Circle(TypedDict, total=True): + radius: int + color: str + x: int + y: int + +def foo(**kwargs: Unpack[Circle]): + ... +foo(x=0, y=0, color='orange') # E: Missing named argument "radius" for "foo" + +class Square(TypedDict, total=False): + side: int + color: str + +def bar(**kwargs: Unpack[Square]): + ... +bar(side=12) +[builtins fixtures/dict.pyi] + +[case testUnpackUnexpectedKeyword] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict, total=False): + name: str + age: int + +def foo(**kwargs: Unpack[Person]) -> None: # N: "foo" defined here + ... +foo(name='John', age=42, department='Sales') # E: Unexpected keyword argument "department" for "foo" +foo(name='Jennifer', age=38) +[builtins fixtures/dict.pyi] + +[case testUnpackKeywordTypes] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]): + ... +foo(name='John', age='42') # E: Argument "age" to "foo" has incompatible type "str"; expected "int" +foo(name='Jennifer', age=38) +[builtins fixtures/dict.pyi] + +[case testUnpackKeywordTypesTypedDict] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +class LegacyPerson(TypedDict): + name: str + age: str + +def foo(**kwargs: Unpack[Person]) -> None: + ... +lp = LegacyPerson(name="test", age="42") +foo(**lp) # E: Argument "age" to "foo" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testFunctionBodyWithUnpackedKwargs] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(**kwargs: Unpack[Person]) -> int: + name: str = kwargs['name'] + age: str = kwargs['age'] # E: Incompatible types in assignment (expression has type "int", variable has type "str") + department: str = kwargs['department'] # E: TypedDict "Person" has no key "department" + return kwargs['age'] +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsOverrides] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +class Base: + def foo(self, **kwargs: Unpack[Person]) -> None: ... +class SubGood(Base): + def foo(self, *, name: str, age: int, extra: bool = False) -> None: ... +class SubBad(Base): + def foo(self, *, name: str, age: str) -> None: ... # E: Argument 2 of "foo" is incompatible with supertype "Base"; supertype defines the argument type as "int" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsOverridesTypedDict] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +class PersonExtra(Person, total=False): + extra: bool + +class Unrelated(TypedDict): + baz: int + +class Base: + def foo(self, **kwargs: Unpack[Person]) -> None: ... +class SubGood(Base): + def foo(self, **kwargs: Unpack[PersonExtra]) -> None: ... +class SubBad(Base): + def foo(self, **kwargs: Unpack[Unrelated]) -> None: ... # E: Signature of "foo" incompatible with supertype "Base" \ + # N: Superclass: \ + # N: def foo(*, name: str, age: int) -> None \ + # N: Subclass: \ + # N: def foo(self, *, baz: int) -> None +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsGeneric] +from typing import Generic, TypeVar +from typing_extensions import Unpack, TypedDict + +T = TypeVar("T") +class Person(TypedDict, Generic[T]): + name: str + value: T + +def foo(**kwargs: Unpack[Person[T]]) -> T: ... +reveal_type(foo(name="test", value=42)) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsInference] +from typing import Generic, TypeVar, Protocol +from typing_extensions import Unpack, TypedDict + +T_contra = TypeVar("T_contra", contravariant=True) +class CBPerson(Protocol[T_contra]): + def __call__(self, **kwargs: Unpack[Person[T_contra]]) -> None: ... + +T = TypeVar("T") +class Person(TypedDict, Generic[T]): + name: str + value: T + +def test(cb: CBPerson[T]) -> T: ... + +def foo(*, name: str, value: int) -> None: ... +reveal_type(test(foo)) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsOverload] +from typing import Any, overload +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +class Fruit(TypedDict): + sort: str + taste: int + +@overload +def foo(**kwargs: Unpack[Person]) -> int: ... +@overload +def foo(**kwargs: Unpack[Fruit]) -> str: ... +def foo(**kwargs: Any) -> Any: + ... + +reveal_type(foo(sort="test", taste=999)) # N: Revealed type is "builtins.str" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsJoin] +from typing_extensions import Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +def foo(*, name: str, age: int) -> None: ... +def bar(**kwargs: Unpack[Person]) -> None: ... + +reveal_type([foo, bar]) # N: Revealed type is "builtins.list[def (*, name: builtins.str, age: builtins.int)]" +reveal_type([bar, foo]) # N: Revealed type is "builtins.list[def (*, name: builtins.str, age: builtins.int)]" +[builtins fixtures/dict.pyi] + +[case testUnpackKwargsParamSpec] +from typing import Callable, Any, TypeVar, List +from typing_extensions import ParamSpec, Unpack, TypedDict + +class Person(TypedDict): + name: str + age: int + +P = ParamSpec('P') +T = TypeVar('T') + +def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... + +@dec +def g(**kwargs: Unpack[Person]) -> int: ... + +reveal_type(g) # N: Revealed type is "def (*, name: builtins.str, age: builtins.int) -> builtins.list[builtins.int]" +[builtins fixtures/dict.pyi] + +[case testUnpackGenericTypedDictImplicitAnyEnabled] +from typing import Generic, TypeVar +from typing_extensions import Unpack, TypedDict + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + key: str + value: T + +def foo(**kwds: Unpack[TD]) -> None: ... # Same as `TD[Any]` +foo(key="yes", value=42) +foo(key="yes", value="ok") +[builtins fixtures/dict.pyi] + +[case testUnpackGenericTypedDictImplicitAnyDisabled] +# flags: --disallow-any-generics +from typing import Generic, TypeVar +from typing_extensions import Unpack, TypedDict + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + key: str + value: T + +def foo(**kwds: Unpack[TD]) -> None: ... # E: Missing type parameters for generic type "TD" +foo(key="yes", value=42) +foo(key="yes", value="ok") +[builtins fixtures/dict.pyi] + +[case testUnpackNoCrashOnEmpty] +from typing_extensions import Unpack + +class C: + def __init__(self, **kwds: Unpack) -> None: ... # E: Unpack[...] requires exactly one type argument +class D: + def __init__(self, **kwds: Unpack[int, str]) -> None: ... # E: Unpack[...] requires exactly one type argument +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 97a9dcaa7410..2ea7f07da3bc 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -130,17 +130,13 @@ two/mod/__init__.py: note: See https://mypy.readthedocs.io/en/stable/running_myp two/mod/__init__.py: note: Common resolutions include: a) using `--exclude` to avoid checking one of them, b) adding `__init__.py` somewhere, c) using `--explicit-package-bases` or adjusting MYPYPATH == Return code: 2 -[case testFlagsFile-skip] +-- Note that we use `----`, because this is how `--` is escaped while `--` is a comment starter. +[case testFlagsFile] # cmd: mypy @flagsfile [file flagsfile] ---always-true=FLAG +----always-true=FLAG main.py [file main.py] -# TODO: this test case passes if you try the exact same thing -# outside of the test suite. what's going on? it's not related -# to the extra flags that testcmdline adds. and things work -# in the test suite with py2 (perhaps because it's a -# special option) x: int FLAG = False if not FLAG: @@ -429,6 +425,7 @@ follow_imports = skip [out] main.py:2: note: Revealed type is "Any" main.py:4: note: Revealed type is "Any" +== Return code: 0 [case testConfigFollowImportsError] # cmd: mypy main.py @@ -521,7 +518,7 @@ reveal_type(missing.x) # Expect Any ignore_missing_imports = True [out] main.py:2: note: Revealed type is "Any" - +== Return code: 0 [case testFailedImportOnWrongCWD] # cmd: mypy main.py @@ -658,15 +655,26 @@ python_version = 3.6 [file int_pow.py] a = 1 b = a + 2 -reveal_type(a**0) # N: Revealed type is "Literal[1]" -reveal_type(a**1) # N: Revealed type is "builtins.int" -reveal_type(a**2) # N: Revealed type is "builtins.int" -reveal_type(a**-0) # N: Revealed type is "Literal[1]" -reveal_type(a**-1) # N: Revealed type is "builtins.float" -reveal_type(a**(-2)) # N: Revealed type is "builtins.float" -reveal_type(a**b) # N: Revealed type is "Any" -reveal_type(a.__pow__(2)) # N: Revealed type is "builtins.int" -reveal_type(a.__pow__(a)) # N: Revealed type is "Any" +reveal_type(a**0) +reveal_type(a**1) +reveal_type(a**2) +reveal_type(a**-0) +reveal_type(a**-1) +reveal_type(a**(-2)) +reveal_type(a**b) +reveal_type(a.__pow__(2)) +reveal_type(a.__pow__(a)) +[out] +int_pow.py:3: note: Revealed type is "Literal[1]" +int_pow.py:4: note: Revealed type is "builtins.int" +int_pow.py:5: note: Revealed type is "builtins.int" +int_pow.py:6: note: Revealed type is "Literal[1]" +int_pow.py:7: note: Revealed type is "builtins.float" +int_pow.py:8: note: Revealed type is "builtins.float" +int_pow.py:9: note: Revealed type is "Any" +int_pow.py:10: note: Revealed type is "builtins.int" +int_pow.py:11: note: Revealed type is "Any" +== Return code: 0 [case testDisallowAnyGenericsBuiltinCollections] # cmd: mypy m.py @@ -1423,7 +1431,6 @@ exclude = (?x)( [out] c/cpkg.py:1: error: "int" not callable - [case testCmdlineTimingStats] # cmd: mypy --timing-stats timing.txt . [file b/__init__.py] @@ -1439,6 +1446,9 @@ b\.c \d+ # cmd: mypy --enable-incomplete-features a.py [file a.py] pass +[out] +Warning: --enable-incomplete-features is deprecated, use --enable-incomplete-feature=FEATURE instead +== Return code: 0 [case testShadowTypingModuleEarlyLoad] # cmd: mypy dir @@ -1478,3 +1488,20 @@ note: A user-defined top-level module with name "typing" is not supported [out] Failed to find builtin module mypy_extensions, perhaps typeshed is broken? == Return code: 2 + +[case testRecursiveAliasesFlagDeprecated] +# cmd: mypy --enable-recursive-aliases a.py +[file a.py] +pass +[out] +Warning: --enable-recursive-aliases is deprecated; recursive types are enabled by default +== Return code: 0 + +[case testNotesOnlyResultInExitSuccess] +# cmd: mypy a.py +[file a.py] +def f(): + x: int = "no" +[out] +a.py:2: note: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs +== Return code: 0 diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 370413ee774b..56966b2f740c 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -185,7 +185,7 @@ Daemon started $ dmypy check foo.py bar.py $ dmypy recheck $ dmypy recheck --update foo.py --remove bar.py sir_not_appearing_in_this_film.py -foo.py:1: error: Import of "bar" ignored +foo.py:1: error: Import of "bar" ignored [misc] foo.py:1: note: (Using --follow-imports=error, module not passed on command line) == Return code: 1 $ dmypy recheck --update bar.py @@ -277,7 +277,7 @@ $ dmypy suggest foo.foo (str) -> int $ {python} -c "import shutil; shutil.copy('foo2.py', 'foo.py')" $ dmypy check foo.py bar.py -bar.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") +bar.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] == Return code: 1 [file foo.py] def foo(arg): @@ -304,7 +304,7 @@ $ dmypy inspect foo:1:2:3:4 Command "inspect" is only valid after a "check" command (that produces no parse errors) == Return code: 2 $ dmypy check foo.py --export-types -foo.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") +foo.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] == Return code: 1 $ dmypy inspect foo:1 Format should be file:line:column[:end_line:end_column] @@ -434,12 +434,12 @@ $ dmypy inspect --show attrs bar.py:10:1 --union-attrs [file foo.py] class B: - def b(self) -> int: ... + def b(self) -> int: return 0 a: int class C(B): a: int y: int - def x(self) -> int: ... + def x(self) -> int: return 0 v: C # line 9 if False: diff --git a/test-data/unit/diff.test b/test-data/unit/diff.test index 7369ea247e26..66adfaecd909 100644 --- a/test-data/unit/diff.test +++ b/test-data/unit/diff.test @@ -1484,3 +1484,16 @@ C = ParamSpec('C') [out] __main__.B __main__.C + +[case testEmptyBodySuper] +from abc import abstractmethod +class C: + @abstractmethod + def meth(self) -> int: ... +[file next.py] +from abc import abstractmethod +class C: + @abstractmethod + def meth(self) -> int: return 0 +[out] +__main__.C.meth diff --git a/test-data/unit/fine-grained-attr.test b/test-data/unit/fine-grained-attr.test index 0a54f9a6ea59..fd7c97da0662 100644 --- a/test-data/unit/fine-grained-attr.test +++ b/test-data/unit/fine-grained-attr.test @@ -21,3 +21,28 @@ class A: [out] == main:5: error: Incompatible return value type (got "Attribute[float]", expected "Attribute[int]") + +[case magicAttributeConsistency] +import m + +[file c.py] +from attr import define + +@define +class A: + a: float + b: int +[builtins fixtures/attr.pyi] + +[file m.py] +from c import A + +A.__attrs_attrs__.a + +[file m.py.2] +from c import A + +A.__attrs_attrs__.b + +[out] +== diff --git a/test-data/unit/fine-grained-blockers.test b/test-data/unit/fine-grained-blockers.test index f3991c0d31e4..a134fb1d4301 100644 --- a/test-data/unit/fine-grained-blockers.test +++ b/test-data/unit/fine-grained-blockers.test @@ -317,8 +317,8 @@ a.py:1: error: invalid syntax == a.py:1: error: invalid syntax == -b.py:3: error: Too many arguments for "f" a.py:3: error: Too many arguments for "g" +b.py:3: error: Too many arguments for "f" [case testDeleteFileWithBlockingError-only_when_nocache] -- Different cache/no-cache tests because: diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test index 4eb55fb125f7..ebe8b86b37ab 100644 --- a/test-data/unit/fine-grained-follow-imports.test +++ b/test-data/unit/fine-grained-follow-imports.test @@ -587,8 +587,8 @@ def f() -> None: main.py:2: error: Cannot find implementation or library stub for module named "p" main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports == -p/m.py:1: error: "str" not callable p/__init__.py:1: error: "int" not callable +p/m.py:1: error: "str" not callable [case testFollowImportsNormalPackageInitFileStub] # flags: --follow-imports=normal @@ -610,11 +610,11 @@ x x x main.py:1: error: Cannot find implementation or library stub for module named "p" main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports == -p/m.pyi:1: error: "str" not callable p/__init__.pyi:1: error: "int" not callable -== p/m.pyi:1: error: "str" not callable +== p/__init__.pyi:1: error: "int" not callable +p/m.pyi:1: error: "str" not callable [case testFollowImportsNormalNamespacePackages] # flags: --follow-imports=normal --namespace-packages @@ -638,12 +638,12 @@ main.py:2: error: Cannot find implementation or library stub for module named "p main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports main.py:2: error: Cannot find implementation or library stub for module named "p2" == -p2/m2.py:1: error: "str" not callable p1/m1.py:1: error: "int" not callable +p2/m2.py:1: error: "str" not callable == +p1/m1.py:1: error: "int" not callable main.py:2: error: Cannot find implementation or library stub for module named "p2.m2" main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports -p1/m1.py:1: error: "int" not callable [case testFollowImportsNormalNewFileOnCommandLine] # flags: --follow-imports=normal @@ -659,8 +659,8 @@ p1/m1.py:1: error: "int" not callable [out] main.py:1: error: "int" not callable == -x.py:1: error: "str" not callable main.py:1: error: "int" not callable +x.py:1: error: "str" not callable [case testFollowImportsNormalSearchPathUpdate-only_when_nocache] # flags: --follow-imports=normal @@ -678,8 +678,8 @@ import bar [out] == -src/bar.py:1: error: "int" not callable src/foo.py:2: error: "str" not callable +src/bar.py:1: error: "int" not callable [case testFollowImportsNormalSearchPathUpdate2-only_when_cache] # flags: --follow-imports=normal diff --git a/test-data/unit/fine-grained-inspect.test b/test-data/unit/fine-grained-inspect.test index 5661c14bc093..a52db3959633 100644 --- a/test-data/unit/fine-grained-inspect.test +++ b/test-data/unit/fine-grained-inspect.test @@ -236,7 +236,7 @@ class C: ... [builtins fixtures/module.pyi] [out] == -{"": ["C", "__annotations__", "__doc__", "__file__", "__name__", "__package__", "bar", "x"], "ModuleType": ["__file__"]} +{"": ["C", "__annotations__", "__doc__", "__file__", "__name__", "__package__", "bar", "x"], "ModuleType": ["__file__", "__getattr__"]} [case testInspectModuleDef] # inspect2: --show=definition --include-kind foo.py:2:1 diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test index a756398fed1f..f76ced64341b 100644 --- a/test-data/unit/fine-grained-modules.test +++ b/test-data/unit/fine-grained-modules.test @@ -38,8 +38,8 @@ def f(x: int) -> None: pass == a.py:2: error: Incompatible return value type (got "int", expected "str") == -b.py:2: error: Too many arguments for "f" a.py:2: error: Incompatible return value type (got "int", expected "str") +b.py:2: error: Too many arguments for "f" == [case testAddFileFixesError] @@ -845,7 +845,7 @@ main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str" == main:1: error: Cannot find implementation or library stub for module named "p.a" main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports -main:1: error: Cannot find implementation or library stub for module named "p" +main:2: error: "object" has no attribute "a" [case testDeletePackage2] import p @@ -2192,17 +2192,55 @@ x = 'x' [case testLibraryStubsNotInstalled] import a [file a.py] -import waitress +import requests [file a.py.2] # nothing [file a.py.3] -import requests +import jack [out] -a.py:1: error: Library stubs not installed for "waitress" (or incompatible with Python 3.7) -a.py:1: note: Hint: "python3 -m pip install types-waitress" +a.py:1: error: Library stubs not installed for "requests" +a.py:1: note: Hint: "python3 -m pip install types-requests" a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports == == -a.py:1: error: Library stubs not installed for "requests" (or incompatible with Python 3.7) -a.py:1: note: Hint: "python3 -m pip install types-requests" +a.py:1: error: Library stubs not installed for "jack" +a.py:1: note: Hint: "python3 -m pip install types-JACK-Client" +a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports + +[case testIgnoreErrorsFromTypeshed] +# flags: --custom-typeshed-dir tmp/ts --follow-imports=normal +# cmd1: mypy a.py +# cmd2: mypy a.py + +[file a.py] +import foobar + +[file ts/stdlib/abc.pyi] +[file ts/stdlib/builtins.pyi] +class object: pass +class str: pass +class ellipsis: pass +[file ts/stdlib/sys.pyi] +[file ts/stdlib/types.pyi] +[file ts/stdlib/typing.pyi] +def cast(x): ... +[file ts/stdlib/typing_extensions.pyi] +[file ts/stdlib/VERSIONS] +[file ts/stubs/mypy_extensions/mypy_extensions.pyi] + +[file ts/stdlib/foobar.pyi.2] +# We report no errors from typeshed. It would be better to test ignoring +# errors from PEP 561 packages, but it's harder to test and uses the +# same code paths, so we are using typeshed instead. +import baz +import zar +undefined + +[file ts/stdlib/baz.pyi.2] +import whatever +undefined + +[out] +a.py:1: error: Cannot find implementation or library stub for module named "foobar" a.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index aa53c6482449..32c4ff2eecf0 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -24,7 +24,7 @@ -- as changed in the initial run with the cache while modules that depended on them -- should be. -- --- Modules that are require a full-module reprocessing by update can be checked with +-- Modules that require a full-module reprocessing by update can be checked with -- [rechecked ...]. This should include any files detected as having changed as well -- as any files that contain targets that need to be reprocessed but which haven't -- been loaded yet. If there is no [rechecked...] directive, it inherits the value of @@ -1814,9 +1814,9 @@ def f() -> Iterator[None]: [out] main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]" == +main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]" a.py:3: error: Cannot find implementation or library stub for module named "b" a.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports -main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]" == main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]" @@ -2968,7 +2968,7 @@ class M(type): pass [out] == -a.py:3: error: Inconsistent metaclass structure for "D" +a.py:3: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [case testFineMetaclassDeclaredUpdate] import a @@ -2984,7 +2984,7 @@ class M(type): pass class M2(type): pass [out] == -a.py:3: error: Inconsistent metaclass structure for "D" +a.py:3: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [case testFineMetaclassRemoveFromClass] import a @@ -3449,7 +3449,6 @@ f(a.x) == [case testNamedTupleUpdate5] -# flags: --enable-recursive-aliases import b [file a.py] from typing import NamedTuple, Optional @@ -3503,7 +3502,7 @@ def foo() -> None: b.py:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") [case testNamedTupleUpdateNonRecursiveToRecursiveFine] -# flags: --enable-recursive-aliases +# flags: --strict-optional import c [file a.py] from b import M @@ -3546,7 +3545,7 @@ c.py:5: error: Incompatible types in assignment (expression has type "Optional[N c.py:7: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" [case testTupleTypeUpdateNonRecursiveToRecursiveFine] -# flags: --enable-recursive-aliases +# flags: --strict-optional import c [file a.py] from b import M @@ -3579,7 +3578,7 @@ c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") [case testTypeAliasUpdateNonRecursiveToRecursiveFine] -# flags: --enable-recursive-aliases +# flags: --strict-optional import c [file a.py] from b import M @@ -3668,7 +3667,6 @@ def foo(x: Point) -> int: b.py:3: error: Unsupported operand types for + ("int" and "str") [case testTypedDictUpdate3] -# flags: --enable-recursive-aliases import b [file a.py] from mypy_extensions import TypedDict @@ -4305,9 +4303,9 @@ y = 0 [file a.py.2] y = '' [out] -main:4: error: Need type annotation for "x" +main:4: error: Need type annotation for "x" (hint: "x: Optional[] = ...") == -main:4: error: Need type annotation for "x" +main:4: error: Need type annotation for "x" (hint: "x: Optional[] = ...") [case testNonePartialType2] import a @@ -4323,9 +4321,9 @@ y = 0 [file a.py.2] y = '' [out] -main:4: error: Need type annotation for "x" +main:4: error: Need type annotation for "x" (hint: "x: Optional[] = ...") == -main:4: error: Need type annotation for "x" +main:4: error: Need type annotation for "x" (hint: "x: Optional[] = ...") [case testNonePartialType3] import a @@ -4337,7 +4335,7 @@ def f() -> None: y = '' [out] == -a.py:1: error: Need type annotation for "y" +a.py:1: error: Need type annotation for "y" (hint: "y: Optional[] = ...") [case testNonePartialType4] import a @@ -4353,7 +4351,7 @@ def f() -> None: global y y = '' [out] -a.py:1: error: Need type annotation for "y" +a.py:1: error: Need type annotation for "y" (hint: "y: Optional[] = ...") == [case testSkippedClass1] @@ -5578,9 +5576,12 @@ import T == main:4: error: "C" expects no type arguments, but 1 given main:4: error: Module "T" is not valid as a type +main:4: note: Perhaps you meant to use a protocol matching the module structure? main:6: error: Free type variable expected in Generic[...] main:7: error: Module "T" is not valid as a type +main:7: note: Perhaps you meant to use a protocol matching the module structure? main:10: error: Module "T" is not valid as a type +main:10: note: Perhaps you meant to use a protocol matching the module structure? main:10: error: Bad number of arguments for type alias, expected: 0, given: 1 [case testChangeClassToModule] @@ -5604,8 +5605,10 @@ import C == == main:3: error: Module "C" is not valid as a type +main:3: note: Perhaps you meant to use a protocol matching the module structure? main:5: error: Module not callable main:8: error: Module "C" is not valid as a type +main:8: note: Perhaps you meant to use a protocol matching the module structure? [case testChangeTypeVarToTypeAlias] @@ -5655,8 +5658,10 @@ import D == == main:3: error: Module "D" is not valid as a type +main:3: note: Perhaps you meant to use a protocol matching the module structure? main:5: error: Module not callable main:8: error: Module "D" is not valid as a type +main:8: note: Perhaps you meant to use a protocol matching the module structure? [case testChangeTypeAliasToModuleUnqualified] @@ -5682,8 +5687,10 @@ import D == == main:3: error: Module "D" is not valid as a type +main:3: note: Perhaps you meant to use a protocol matching the module structure? main:5: error: Module not callable main:8: error: Module "D" is not valid as a type +main:8: note: Perhaps you meant to use a protocol matching the module structure? [case testChangeFunctionToVariableAndRefreshUsingStaleDependency] import a @@ -7942,7 +7949,7 @@ class Foo(a.I): == [case testImplicitOptionalRefresh1] -# flags: --strict-optional +# flags: --strict-optional --implicit-optional from x import f def foo(x: int = None) -> None: f() @@ -8691,8 +8698,8 @@ main:2: note: Revealed type is "builtins.int" == main:2: note: Revealed type is "Literal[1]" == -mod.py:2: error: Incompatible types in assignment (expression has type "Literal[2]", variable has type "Literal[1]") main:2: note: Revealed type is "Literal[1]" +mod.py:2: error: Incompatible types in assignment (expression has type "Literal[2]", variable has type "Literal[1]") [case testLiteralFineGrainedFunctionConversion] from mod import foo @@ -9180,10 +9187,10 @@ a.py:1: error: Type signature has too few arguments a.py:5: error: Type signature has too few arguments a.py:11: error: Type signature has too few arguments == +c.py:1: error: Type signature has too few arguments a.py:1: error: Type signature has too few arguments a.py:5: error: Type signature has too few arguments a.py:11: error: Type signature has too few arguments -c.py:1: error: Type signature has too few arguments [case testErrorReportingNewAnalyzer] # flags: --disallow-any-generics @@ -9795,11 +9802,11 @@ class ExampleClass(Generic[T]): [case testDataclassCheckTypeVarBoundsInReprocess] # flags: --python-version 3.7 from dataclasses import dataclass -from typing import Protocol, Dict, TypeVar, Generic +from typing import ClassVar, Protocol, Dict, TypeVar, Generic from m import x class DataclassProtocol(Protocol): - __dataclass_fields__: Dict + __dataclass_fields__: ClassVar[Dict] T = TypeVar("T", bound=DataclassProtocol) @@ -9818,3 +9825,308 @@ x: str [builtins fixtures/dataclasses.pyi] [out] == + +[case testParamSpecCached] +import a + +[file a.py] +import b + +def f(x: int) -> str: return 'x' + +b.foo(f) + +[file a.py.2] +import b + +def f(x: int) -> str: return 'x' + +reveal_type(b.foo(f)) + +[file b.py] +from typing import TypeVar, Callable, Union +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") + +def foo(f: Callable[P, T]) -> Callable[P, Union[T, None]]: + return f + +[file b.py.2] +from typing import TypeVar, Callable, Union +from typing_extensions import ParamSpec + +P = ParamSpec("P") +T = TypeVar("T") + +def foo(f: Callable[P, T]) -> Callable[P, Union[T, None]]: + return f + +x = 0 # Arbitrary change to trigger reprocessing + +[builtins fixtures/dict.pyi] +[out] +== +a.py:5: note: Revealed type is "def (x: builtins.int) -> builtins.str" + +[case testTypeVarTupleCached] +import a + +[file a.py] +import b + +def f(x: int) -> str: return 'x' + +b.foo((1, 'x')) + +[file a.py.2] +import b + +reveal_type(b.foo((1, 'x'))) + +[file b.py] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +def foo(t: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + return t + +[file b.py.2] +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") + +def foo(t: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + return t + +x = 0 # Arbitrary change to trigger reprocessing +[builtins fixtures/dict.pyi] +[out] +== +a.py:3: note: Revealed type is "Tuple[Literal[1]?, Literal['x']?]" + +[case testUnpackKwargsUpdateFine] +import m +[file shared.py] +from typing_extensions import TypedDict + +class Person(TypedDict): + name: str + age: int + +[file shared.py.2] +from typing_extensions import TypedDict + +class Person(TypedDict): + name: str + age: str + +[file lib.py] +from typing_extensions import Unpack +from shared import Person + +def foo(**kwargs: Unpack[Person]): + ... +[file m.py] +from lib import foo +foo(name='Jennifer', age=38) + +[builtins fixtures/dict.pyi] +[out] +== +m.py:2: error: Argument "age" to "foo" has incompatible type "int"; expected "str" + +[case testModuleAsProtocolImplementationFine] +import m +[file m.py] +from typing import Protocol +from lib import C + +class Options(Protocol): + timeout: int + def update(self) -> bool: ... + +def setup(options: Options) -> None: ... +setup(C().config) + +[file lib.py] +import default_config + +class C: + config = default_config + +[file default_config.py] +timeout = 100 +def update() -> bool: ... + +[file default_config.py.2] +timeout = 100 +def update() -> str: ... +[builtins fixtures/module.pyi] +[out] +== +m.py:9: error: Argument 1 to "setup" has incompatible type Module; expected "Options" +m.py:9: note: Following member(s) of "Module default_config" have conflicts: +m.py:9: note: Expected: +m.py:9: note: def update() -> bool +m.py:9: note: Got: +m.py:9: note: def update() -> str + +[case testBoundGenericMethodFine] +import main +[file main.py] +import lib +[file main.py.3] +import lib +reveal_type(lib.foo(42)) +[file lib/__init__.pyi] +from lib import context +foo = context.test.foo +[file lib/context.pyi] +from typing import TypeVar +import lib.other + +T = TypeVar("T") +class Test: + def foo(self, x: T, n: lib.other.C = ...) -> T: ... +test: Test + +[file lib/other.pyi] +class C: ... +[file lib/other.pyi.2] +class B: ... +class C(B): ... +[out] +== +== +main.py:2: note: Revealed type is "builtins.int" + +[case testBoundGenericMethodParamSpecFine] +import main +[file main.py] +import lib +[file main.py.3] +from typing import Callable +import lib +f: Callable[[], int] +reveal_type(lib.foo(f)) +[file lib/__init__.pyi] +from lib import context +foo = context.test.foo +[file lib/context.pyi] +from typing_extensions import ParamSpec +from typing import Callable +import lib.other + +P = ParamSpec("P") +class Test: + def foo(self, x: Callable[P, int], n: lib.other.C = ...) -> Callable[P, str]: ... +test: Test + +[file lib/other.pyi] +class C: ... +[file lib/other.pyi.2] +class B: ... +class C(B): ... +[builtins fixtures/dict.pyi] +[out] +== +== +main.py:4: note: Revealed type is "def () -> builtins.str" + +[case testAbstractBodyTurnsEmpty] +# flags: --strict-optional +from b import Base + +class Sub(Base): + def meth(self) -> int: + return super().meth() + +[file b.py] +from abc import abstractmethod +class Base: + @abstractmethod + def meth(self) -> int: return 0 + +[file b.py.2] +from abc import abstractmethod +class Base: + @abstractmethod + def meth(self) -> int: ... +[out] +== +main:6: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe + +[case testAbstractBodyTurnsEmptyProtocol] +# flags: --strict-optional +from b import Base + +class Sub(Base): + def meth(self) -> int: + return super().meth() + +[file b.py] +from typing import Protocol +class Base(Protocol): + def meth(self) -> int: return 0 +[file b.py.2] +from typing import Protocol +class Base(Protocol): + def meth(self) -> int: ... +[out] +== +main:6: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe + +[case testPrettyMessageSorting] +# flags: --pretty +import a + +[file a.py] +1 + '' +import b + +[file b.py] +object + 1 + +[file b.py.2] +object + 1 +1() + +[out] +b.py:1: error: Unsupported left operand type for + ("Type[object]") + object + 1 + ^ +a.py:1: error: Unsupported operand types for + ("int" and "str") + 1 + '' + ^ +== +b.py:1: error: Unsupported left operand type for + ("Type[object]") + object + 1 + ^ +b.py:2: error: "int" not callable + 1() + ^ +a.py:1: error: Unsupported operand types for + ("int" and "str") + 1 + '' + ^ +[out version>=3.8] +b.py:1: error: Unsupported left operand type for + ("Type[object]") + object + 1 + ^~~~~~~~~~ +a.py:1: error: Unsupported operand types for + ("int" and "str") + 1 + '' + ^~ +== +b.py:1: error: Unsupported left operand type for + ("Type[object]") + object + 1 + ^~~~~~~~~~ +b.py:2: error: "int" not callable + 1() + ^~~ +a.py:1: error: Unsupported operand types for + ("int" and "str") + 1 + '' + ^~ diff --git a/test-data/unit/fixtures/exception.pyi b/test-data/unit/fixtures/exception.pyi index bf6d21c8716e..1c88723e7191 100644 --- a/test-data/unit/fixtures/exception.pyi +++ b/test-data/unit/fixtures/exception.pyi @@ -1,3 +1,4 @@ +import sys from typing import Generic, TypeVar T = TypeVar('T') @@ -5,7 +6,8 @@ class object: def __init__(self): pass class type: pass -class tuple(Generic[T]): pass +class tuple(Generic[T]): + def __ge__(self, other: object) -> bool: ... class function: pass class int: pass class str: pass @@ -13,11 +15,14 @@ class unicode: pass class bool: pass class ellipsis: pass -# Note: this is a slight simplification. In Python 2, the inheritance hierarchy -# is actually Exception -> StandardError -> RuntimeError -> ... class BaseException: def __init__(self, *args: object) -> None: ... class Exception(BaseException): pass class RuntimeError(Exception): pass class NotImplementedError(RuntimeError): pass +if sys.version_info >= (3, 11): + _BT_co = TypeVar("_BT_co", bound=BaseException, covariant=True) + _T_co = TypeVar("_T_co", bound=Exception, covariant=True) + class BaseExceptionGroup(BaseException, Generic[_BT_co]): ... + class ExceptionGroup(BaseExceptionGroup[_T_co], Exception): ... diff --git a/test-data/unit/fixtures/isinstance.pyi b/test-data/unit/fixtures/isinstance.pyi index 7f7cf501b5de..aa8bfce7fbe0 100644 --- a/test-data/unit/fixtures/isinstance.pyi +++ b/test-data/unit/fixtures/isinstance.pyi @@ -14,6 +14,7 @@ class function: pass def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass def issubclass(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass +def hasattr(x: object, name: str) -> bool: pass class int: def __add__(self, other: 'int') -> 'int': pass diff --git a/test-data/unit/fixtures/module.pyi b/test-data/unit/fixtures/module.pyi index ac1d3688ed12..47408befd5ce 100644 --- a/test-data/unit/fixtures/module.pyi +++ b/test-data/unit/fixtures/module.pyi @@ -19,3 +19,5 @@ class ellipsis: pass classmethod = object() staticmethod = object() +property = object() +def hasattr(x: object, name: str) -> bool: pass diff --git a/test-data/unit/fixtures/primitives.pyi b/test-data/unit/fixtures/primitives.pyi index c72838535443..9553df4b40c7 100644 --- a/test-data/unit/fixtures/primitives.pyi +++ b/test-data/unit/fixtures/primitives.pyi @@ -57,10 +57,7 @@ class function: pass class ellipsis: pass class range(Sequence[int]): - @overload - def __init__(self, stop: int) -> None: pass - @overload - def __init__(self, start: int, stop: int, step: int = ...) -> None: pass + def __init__(self, __x: int, __y: int = ..., __z: int = ...) -> None: pass def count(self, value: int) -> int: pass def index(self, value: int) -> int: pass def __getitem__(self, i: int) -> int: pass diff --git a/test-data/unit/fixtures/tuple.pyi b/test-data/unit/fixtures/tuple.pyi index 6f40356bb5f0..5c69a4ad1eb5 100644 --- a/test-data/unit/fixtures/tuple.pyi +++ b/test-data/unit/fixtures/tuple.pyi @@ -1,6 +1,6 @@ # Builtins stub used in tuple-related test cases. -from typing import Iterable, Iterator, TypeVar, Generic, Sequence, Any, overload, Tuple, Type +from typing import Iterable, Iterator, TypeVar, Generic, Sequence, Optional, overload, Tuple, Type T = TypeVar("T") Tco = TypeVar('Tco', covariant=True) @@ -35,6 +35,7 @@ class slice: pass class bool(int): pass class str: pass # For convenience class bytes: pass +class bytearray: pass class unicode: pass class list(Sequence[T], Generic[T]): @@ -47,6 +48,6 @@ class list(Sequence[T], Generic[T]): def isinstance(x: object, t: type) -> bool: pass -def sum(iterable: Iterable[T], start: T = None) -> T: pass +def sum(iterable: Iterable[T], start: Optional[T] = None) -> T: pass class BaseException: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index dad30dd7bcee..c406da986818 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -160,8 +160,8 @@ class SupportsAbs(Protocol[T_co]): def runtime_checkable(cls: T) -> T: return cls -class ContextManager(Generic[T]): - def __enter__(self) -> T: pass +class ContextManager(Generic[T_co]): + def __enter__(self) -> T_co: pass # Use Any because not all the precise types are in the fixtures. def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi index 3404dc69de44..d51134ead599 100644 --- a/test-data/unit/fixtures/typing-namedtuple.pyi +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -4,6 +4,7 @@ Any = 0 overload = 0 Type = 0 Literal = 0 +Optional = 0 T_co = TypeVar('T_co', covariant=True) KT = TypeVar('KT') diff --git a/test-data/unit/lib-stub/abc.pyi b/test-data/unit/lib-stub/abc.pyi index da90b588fca3..e60f709a5187 100644 --- a/test-data/unit/lib-stub/abc.pyi +++ b/test-data/unit/lib-stub/abc.pyi @@ -2,8 +2,8 @@ from typing import Type, Any, TypeVar T = TypeVar('T', bound=Type[Any]) -class ABC(type): pass class ABCMeta(type): def register(cls, tp: T) -> T: pass +class ABC(metaclass=ABCMeta): pass abstractmethod = object() abstractproperty = object() diff --git a/test-data/unit/lib-stub/contextlib.pyi b/test-data/unit/lib-stub/contextlib.pyi index e7db25da1b5f..e2a0cccd562a 100644 --- a/test-data/unit/lib-stub/contextlib.pyi +++ b/test-data/unit/lib-stub/contextlib.pyi @@ -7,6 +7,7 @@ _T = TypeVar('_T') class GeneratorContextManager(ContextManager[_T], Generic[_T]): def __call__(self, func: Callable[..., _T]) -> Callable[..., _T]: ... +# This does not match `typeshed` definition, needs `ParamSpec`: def contextmanager(func: Callable[..., Iterator[_T]]) -> Callable[..., GeneratorContextManager[_T]]: ... diff --git a/test-data/unit/lib-stub/types.pyi b/test-data/unit/lib-stub/types.pyi index 4a6093f701cc..012fd8503377 100644 --- a/test-data/unit/lib-stub/types.pyi +++ b/test-data/unit/lib-stub/types.pyi @@ -1,4 +1,4 @@ -from typing import TypeVar +from typing import Any, TypeVar import sys _T = TypeVar('_T') @@ -6,7 +6,8 @@ _T = TypeVar('_T') def coroutine(func: _T) -> _T: pass class ModuleType: - __file__ = ... # type: str + __file__: str + def __getattr__(self, name: str) -> Any: pass if sys.version_info >= (3, 10): class Union: diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 0a1bb42b936c..23d97704d934 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -27,6 +27,7 @@ NoReturn = 0 Never = 0 NewType = 0 ParamSpec = 0 +TYPE_CHECKING = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi index b82b73d49a71..e92f7e913502 100644 --- a/test-data/unit/lib-stub/typing_extensions.pyi +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -10,6 +10,9 @@ class _SpecialForm: def __getitem__(self, typeargs: Any) -> Any: pass + def __call__(self, arg: Any) -> Any: + pass + NamedTuple = 0 Protocol: _SpecialForm = ... def runtime_checkable(x: _T) -> _T: pass diff --git a/test-data/unit/plugins/add_classmethod.py b/test-data/unit/plugins/add_classmethod.py new file mode 100644 index 000000000000..5aacc69a8f01 --- /dev/null +++ b/test-data/unit/plugins/add_classmethod.py @@ -0,0 +1,28 @@ +from typing import Callable, Optional + +from mypy.nodes import ARG_POS, Argument, Var +from mypy.plugin import ClassDefContext, Plugin +from mypy.plugins.common import add_method +from mypy.types import NoneType + + +class ClassMethodPlugin(Plugin): + def get_base_class_hook(self, fullname: str) -> Optional[Callable[[ClassDefContext], None]]: + if "BaseAddMethod" in fullname: + return add_extra_methods_hook + return None + + +def add_extra_methods_hook(ctx: ClassDefContext) -> None: + add_method(ctx, "foo_classmethod", [], NoneType(), is_classmethod=True) + add_method( + ctx, + "foo_staticmethod", + [Argument(Var(""), ctx.api.named_type("builtins.int"), None, ARG_POS)], + ctx.api.named_type("builtins.str"), + is_staticmethod=True, + ) + + +def plugin(version): + return ClassMethodPlugin diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 028d2aff561f..59ab586b17e6 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -660,8 +660,8 @@ a + 1 [out] _testMapStr.py:4: error: No overload variant of "__add__" of "list" matches argument type "int" _testMapStr.py:4: note: Possible overload variants: -_testMapStr.py:4: note: def __add__(self, List[str]) -> List[str] -_testMapStr.py:4: note: def [_S] __add__(self, List[_S]) -> List[Union[_S, str]] +_testMapStr.py:4: note: def __add__(self, List[str], /) -> List[str] +_testMapStr.py:4: note: def [_S] __add__(self, List[_S], /) -> List[Union[_S, str]] [case testRelativeImport] import typing @@ -805,8 +805,8 @@ t + 1 [out] _program.py:3: error: No overload variant of "__add__" of "tuple" matches argument type "int" _program.py:3: note: Possible overload variants: -_program.py:3: note: def __add__(self, Tuple[str, ...]) -> Tuple[str, ...] -_program.py:3: note: def [_T] __add__(self, Tuple[_T, ...]) -> Tuple[Union[str, _T], ...] +_program.py:3: note: def __add__(self, Tuple[str, ...], /) -> Tuple[str, ...] +_program.py:3: note: def [_T] __add__(self, Tuple[_T, ...], /) -> Tuple[Union[str, _T], ...] [case testMultiplyTupleByIntegerReverse] n = 4 @@ -815,8 +815,8 @@ t + 1 [out] _program.py:3: error: No overload variant of "__add__" of "tuple" matches argument type "int" _program.py:3: note: Possible overload variants: -_program.py:3: note: def __add__(self, Tuple[str, ...]) -> Tuple[str, ...] -_program.py:3: note: def [_T] __add__(self, Tuple[_T, ...]) -> Tuple[Union[str, _T], ...] +_program.py:3: note: def __add__(self, Tuple[str, ...], /) -> Tuple[str, ...] +_program.py:3: note: def [_T] __add__(self, Tuple[_T, ...], /) -> Tuple[Union[str, _T], ...] [case testDictWithKeywordArgs] from typing import Dict, Any, List @@ -1099,8 +1099,8 @@ _testTypedDictGet.py:8: note: Revealed type is "builtins.str" _testTypedDictGet.py:9: note: Revealed type is "builtins.object" _testTypedDictGet.py:10: error: All overload variants of "get" of "Mapping" require at least one argument _testTypedDictGet.py:10: note: Possible overload variants: -_testTypedDictGet.py:10: note: def get(self, str) -> object -_testTypedDictGet.py:10: note: def [_T] get(self, str, default: object) -> object +_testTypedDictGet.py:10: note: def get(self, str, /) -> object +_testTypedDictGet.py:10: note: def [_T] get(self, str, /, default: object) -> object _testTypedDictGet.py:12: note: Revealed type is "builtins.object" [case testTypedDictMappingMethods] @@ -1130,9 +1130,9 @@ _testTypedDictMappingMethods.py:5: note: Revealed type is "builtins.str" _testTypedDictMappingMethods.py:6: note: Revealed type is "typing.Iterator[builtins.str]" _testTypedDictMappingMethods.py:7: note: Revealed type is "builtins.int" _testTypedDictMappingMethods.py:8: note: Revealed type is "builtins.bool" -_testTypedDictMappingMethods.py:9: note: Revealed type is "typing.KeysView[builtins.str]" -_testTypedDictMappingMethods.py:10: note: Revealed type is "typing.ItemsView[builtins.str, builtins.object]" -_testTypedDictMappingMethods.py:11: note: Revealed type is "typing.ValuesView[builtins.object]" +_testTypedDictMappingMethods.py:9: note: Revealed type is "_collections_abc.dict_keys[builtins.str, builtins.object]" +_testTypedDictMappingMethods.py:10: note: Revealed type is "_collections_abc.dict_items[builtins.str, builtins.object]" +_testTypedDictMappingMethods.py:11: note: Revealed type is "_collections_abc.dict_values[builtins.str, builtins.object]" _testTypedDictMappingMethods.py:12: note: Revealed type is "TypedDict('_testTypedDictMappingMethods.Cell', {'value': builtins.int})" _testTypedDictMappingMethods.py:13: note: Revealed type is "builtins.int" _testTypedDictMappingMethods.py:15: error: Unexpected TypedDict key "invalid" @@ -1561,8 +1561,9 @@ import scribe # No Python 3 stubs available for scribe from scribe import x import maxminddb # Python 3 stubs available for maxminddb import foobar_asdf +import jack # This has a stubs package but was never bundled with mypy, so ignoring works [out] -_testIgnoreImportIfNoPython3StubAvailable.py:4: error: Library stubs not installed for "maxminddb" (or incompatible with Python 3.7) +_testIgnoreImportIfNoPython3StubAvailable.py:4: error: Library stubs not installed for "maxminddb" _testIgnoreImportIfNoPython3StubAvailable.py:4: note: Hint: "python3 -m pip install types-maxminddb" _testIgnoreImportIfNoPython3StubAvailable.py:4: note: (or run "mypy --install-types" to install all missing stub packages) _testIgnoreImportIfNoPython3StubAvailable.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports @@ -1574,7 +1575,7 @@ import maxminddb [out] _testNoPython3StubAvailable.py:1: error: Cannot find implementation or library stub for module named "scribe" _testNoPython3StubAvailable.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports -_testNoPython3StubAvailable.py:3: error: Library stubs not installed for "maxminddb" (or incompatible with Python 3.7) +_testNoPython3StubAvailable.py:3: error: Library stubs not installed for "maxminddb" _testNoPython3StubAvailable.py:3: note: Hint: "python3 -m pip install types-maxminddb" _testNoPython3StubAvailable.py:3: note: (or run "mypy --install-types" to install all missing stub packages) @@ -1620,7 +1621,6 @@ _testEnumValueWithPlaceholderNodeType.py:6: error: Incompatible types in assignm _testEnumValueWithPlaceholderNodeType.py:6: error: Name "Missing" is not defined [case testTypeshedRecursiveTypesExample] -# flags: --enable-recursive-aliases from typing import List, Union Recursive = Union[str, List["Recursive"]] diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 943420fa98a1..2b10beacbf97 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -316,15 +316,16 @@ x = y tmp/k.py:2: error: Name "y" is not defined [case testPackageWithoutInitFile] +# flags: --no-namespace-packages import typing import m.n m.n.x [file m/n.py] x = 1 [out] -main:2: error: Cannot find implementation or library stub for module named "m.n" -main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports -main:2: error: Cannot find implementation or library stub for module named "m" +main:3: error: Cannot find implementation or library stub for module named "m.n" +main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +main:3: error: Cannot find implementation or library stub for module named "m" [case testBreakOutsideLoop] break @@ -1456,9 +1457,11 @@ bad: Tuple[Unpack[int]] # E: builtins.int cannot be unpacked (must be tuple or [builtins fixtures/tuple.pyi] [case testTypeVarTuple] +from typing import Generic from typing_extensions import TypeVarTuple, Unpack TVariadic = TypeVarTuple('TVariadic') +TVariadic2 = TypeVarTuple('TVariadic2') TP = TypeVarTuple('?') # E: String argument 1 "?" to TypeVarTuple(...) does not match variable name "TP" TP2: int = TypeVarTuple('TP2') # E: Cannot declare the type of a TypeVar or similar construct TP3 = TypeVarTuple() # E: Too few arguments for TypeVarTuple() @@ -1467,3 +1470,7 @@ TP5 = TypeVarTuple(t='TP5') # E: TypeVarTuple() expects a string literal as fir x: TVariadic # E: TypeVarTuple "TVariadic" is unbound y: Unpack[TVariadic] # E: TypeVarTuple "TVariadic" is unbound + + +class Variadic(Generic[Unpack[TVariadic], Unpack[TVariadic2]]): # E: Can only use one type var tuple in a class def + pass diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 408f116443d2..8467271e5593 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -2705,3 +2705,32 @@ def f(): return 0 [out] def f(): ... + +[case testKnownMagicMethodsReturnTypes] +class Some: + def __len__(self): ... + def __length_hint__(self): ... + def __init__(self): ... + def __del__(self): ... + def __bool__(self): ... + def __bytes__(self): ... + def __format__(self, spec): ... + def __contains__(self, obj): ... + def __complex__(self): ... + def __int__(self): ... + def __float__(self): ... + def __index__(self): ... +[out] +class Some: + def __len__(self) -> int: ... + def __length_hint__(self) -> int: ... + def __init__(self) -> None: ... + def __del__(self) -> None: ... + def __bool__(self) -> bool: ... + def __bytes__(self) -> bytes: ... + def __format__(self, spec) -> str: ... + def __contains__(self, obj) -> bool: ... + def __complex__(self) -> complex: ... + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __index__(self) -> int: ... diff --git a/test-requirements.txt b/test-requirements.txt index d5bc3f1113a9..7fe486387f2f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,12 +1,11 @@ -r mypy-requirements.txt -r build-requirements.txt attrs>=18.0 -black==22.6.0 # must match version in .pre-commit-config.yaml -filelock>=3.3.0,<3.4.2; python_version<'3.7' -filelock>=3.3.0; python_version>='3.7' -flake8==3.9.2 # must match version in .pre-commit-config.yaml -flake8-bugbear==22.7.1 # must match version in .pre-commit-config.yaml -flake8-noqa==1.2.8 # must match version in .pre-commit-config.yaml +black==22.10.0 # must match version in .pre-commit-config.yaml +filelock>=3.3.0 +flake8==5.0.4 # must match version in .pre-commit-config.yaml +flake8-bugbear==22.9.23 # must match version in .pre-commit-config.yaml +flake8-noqa==1.2.9 # must match version in .pre-commit-config.yaml isort[colors]==5.10.1 # must match version in .pre-commit-config.yaml lxml>=4.4.0; python_version<'3.11' psutil>=4.0 @@ -16,7 +15,5 @@ pytest-xdist>=1.34.0 pytest-forked>=1.3.0,<2.0.0 pytest-cov>=2.10.0 py>=1.5.2 -typed_ast>=1.5.4,<2; python_version>='3.8' setuptools!=50 six -importlib-metadata>=4.6.1,<5.0.0 diff --git a/tox.ini b/tox.ini index d2284813195e..92810bed9981 100644 --- a/tox.ini +++ b/tox.ini @@ -14,36 +14,9 @@ isolated_build = true [testenv] description = run the test driver with {basepython} -setenv = cov: COVERAGE_FILE={toxworkdir}/.coverage.{envname} passenv = PYTEST_XDIST_WORKER_COUNT PROGRAMDATA PROGRAMFILES(X86) deps = -rtest-requirements.txt -commands = python -m pytest --durations 100 {posargs} - cov: python -m pytest --durations 100 {posargs: --cov mypy --cov-config setup.cfg} - - -[testenv:coverage] -description = [run locally after tests]: combine coverage data and create report -deps = - coverage >= 4.5.1, < 5 - diff_cover >= 1.0.5, <2 -skip_install = True -passenv = - {[testenv]passenv} - DIFF_AGAINST -setenv = COVERAGE_FILE={toxworkdir}/.coverage -commands = - coverage combine --rcfile setup.cfg - coverage report -m --rcfile setup.cfg - coverage xml -o {toxworkdir}/coverage.xml --rcfile setup.cfg - coverage html -d {toxworkdir}/htmlcov --rcfile setup.cfg - diff-cover --compare-branch {env:DIFF_AGAINST:origin/master} {toxworkdir}/coverage.xml -depends = - py36, - py37, - py38, - py39, - py310, -parallel_show_output = True +commands = python -m pytest {posargs} [testenv:lint] description = check the code style @@ -56,7 +29,7 @@ commands = description = type check ourselves commands = python -m mypy --config-file mypy_self_check.ini -p mypy -p mypyc - python -m mypy --config-file mypy_self_check.ini misc/proper_plugin.py + python -m mypy --config-file mypy_self_check.ini misc --exclude misc/fix_annotate.py --exclude misc/async_matrix.py --exclude misc/sync-typeshed.py [testenv:docs] description = invoke sphinx-build to build the HTML docs