From a1a22ca50f29484ef677d011fd8905cd0ad20f46 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 10 Feb 2021 08:25:49 -0800 Subject: [PATCH 01/12] Add --exclude (#9992) Resolves #4675, resolves #9981. Additionally, we always ignore site-packages and node_modules, and directories starting with a dot. Also note that this doesn't really affect import discovery; it only directly affects passing files or packages to mypy. The additional check before suggesting "are you missing an __init__.py" didn't make any sense to me, so I removed it, appended to the message and downgraded the severity to note. Co-authored-by: hauntsaninja <> --- docs/source/command_line.rst | 24 ++++++ docs/source/config_file.rst | 12 +++ docs/source/running_mypy.rst | 3 +- mypy/build.py | 16 ++-- mypy/find_sources.py | 11 ++- mypy/main.py | 9 +++ mypy/modulefinder.py | 25 ++++++- mypy/options.py | 2 + mypy/test/test_find_sources.py | 131 ++++++++++++++++++++++++++++++--- mypy_self_check.ini | 1 + test-data/unit/cmdline.test | 16 +--- 11 files changed, 210 insertions(+), 40 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 40df775742a6..db4da1436189 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -49,6 +49,30 @@ for full details, see :ref:`running-mypy`. Asks mypy to type check the provided string as a program. +.. option:: --exclude + + A regular expression that matches file names, directory names and paths + which mypy should ignore while recursively discovering files to check. + Use forward slashes on all platforms. + + For instance, to avoid discovering any files named `setup.py` you could + pass ``--exclude '/setup\.py$'``. Similarly, you can ignore discovering + directories with a given name by e.g. ``--exclude /build/`` or + those matching a subpath with ``--exclude /project/vendor/``. + + Note that this flag only affects recursive discovery, that is, when mypy is + discovering files within a directory tree or submodules of a package to + check. If you pass a file or module explicitly it will still be checked. For + instance, ``mypy --exclude '/setup.py$' but_still_check/setup.py``. + + Note that mypy will never recursively discover files and directories named + "site-packages", "node_modules" or "__pycache__", or those whose name starts + with a period, exactly as ``--exclude + '/(site-packages|node_modules|__pycache__|\..*)/$'`` would. Mypy will also + never recursively discover files with extensions other than ``.py`` or + ``.pyi``. + + Optional arguments ****************** diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 11aa73fbf5d0..6ae02fe8aa52 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -192,6 +192,18 @@ section of the command line docs. This option may only be set in the global section (``[mypy]``). +.. confval:: exclude + + :type: regular expression + + A regular expression that matches file names, directory names and paths + which mypy should ignore while recursively discovering files to check. + Use forward slashes on all platforms. + + For more details, see :option:`--exclude `. + + This option may only be set in the global section (``[mypy]``). + .. confval:: namespace_packages :type: boolean diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index 3d5b9ff6d17a..2c1d14b6d858 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -355,7 +355,8 @@ to modules to type check. - Mypy will check all paths provided that correspond to files. - Mypy will recursively discover and check all files ending in ``.py`` or - ``.pyi`` in directory paths provided. + ``.pyi`` in directory paths provided, after accounting for + :option:`--exclude `. - For each file to be checked, mypy will attempt to associate the file (e.g. ``project/foo/bar/baz.py``) with a fully qualified module name (e.g. diff --git a/mypy/build.py b/mypy/build.py index e6f597af31bc..324a8f853456 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -15,7 +15,6 @@ import gc import json import os -import pathlib import re import stat import sys @@ -2552,6 +2551,7 @@ def log_configuration(manager: BuildManager, sources: List[BuildSource]) -> None ("Current Executable", sys.executable), ("Cache Dir", manager.options.cache_dir), ("Compiled", str(not __file__.endswith(".py"))), + ("Exclude", manager.options.exclude), ] for conf_name, conf_value in configuration_vars: @@ -2751,14 +2751,12 @@ def load_graph(sources: List[BuildSource], manager: BuildManager, "Duplicate module named '%s' (also at '%s')" % (st.id, graph[st.id].xpath), blocker=True, ) - p1 = len(pathlib.PurePath(st.xpath).parents) - p2 = len(pathlib.PurePath(graph[st.id].xpath).parents) - - if p1 != p2: - manager.errors.report( - -1, -1, - "Are you missing an __init__.py?" - ) + manager.errors.report( + -1, -1, + "Are you missing an __init__.py? Alternatively, consider using --exclude to " + "avoid checking one of them.", + severity='note' + ) manager.errors.raise_error() graph[st.id] = st diff --git a/mypy/find_sources.py b/mypy/find_sources.py index 47d686cddcbc..4f50d8ff52b2 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -6,7 +6,7 @@ from typing import List, Sequence, Set, Tuple, Optional from typing_extensions import Final -from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path +from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_exclude from mypy.fscache import FileSystemCache from mypy.options import Options @@ -91,6 +91,8 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None: self.fscache = fscache self.explicit_package_bases = get_explicit_package_bases(options) self.namespace_packages = options.namespace_packages + self.exclude = options.exclude + self.verbosity = options.verbosity def is_explicit_package_base(self, path: str) -> bool: assert self.explicit_package_bases @@ -103,10 +105,15 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(path), key=keyfunc) for name in names: # Skip certain names altogether - if name == '__pycache__' or name.startswith('.') or name.endswith('~'): + if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): continue subpath = os.path.join(path, name) + if matches_exclude( + subpath, self.exclude, self.fscache, self.verbosity >= 2 + ): + continue + if self.fscache.isdir(subpath): sub_sources = self.find_sources_in_dir(subpath) if sub_sources: diff --git a/mypy/main.py b/mypy/main.py index ab38f7478b3f..ea68ee41d0f2 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -791,6 +791,15 @@ def add_invertible_flag(flag: str, code_group.add_argument( '--explicit-package-bases', action='store_true', help="Use current directory and MYPYPATH to determine module names of files passed") + code_group.add_argument( + "--exclude", + metavar="PATTERN", + default="", + help=( + "Regular expression to match file names, directory names or paths which mypy should " + "ignore while recursively discovering files to check, e.g. --exclude '/setup\\.py$'" + ) + ) code_group.add_argument( '-m', '--module', action='append', metavar='MODULE', default=[], diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index bdc71d7a7e58..2c708b8f802d 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -7,6 +7,7 @@ import collections import functools import os +import re import subprocess import sys from enum import Enum @@ -380,10 +381,15 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(package_path)) for name in names: # Skip certain names altogether - if name == '__pycache__' or name.startswith('.') or name.endswith('~'): + if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): continue subpath = os.path.join(package_path, name) + if self.options and matches_exclude( + subpath, self.options.exclude, self.fscache, self.options.verbosity >= 2 + ): + continue + if self.fscache.isdir(subpath): # Only recurse into packages if (self.options and self.options.namespace_packages) or ( @@ -397,13 +403,26 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: if stem == '__init__': continue if stem not in seen and '.' not in stem and suffix in PYTHON_EXTENSIONS: - # (If we sorted names) we could probably just make the BuildSource ourselves, - # but this ensures compatibility with find_module / the cache + # (If we sorted names by keyfunc) we could probably just make the BuildSource + # ourselves, but this ensures compatibility with find_module / the cache seen.add(stem) sources.extend(self.find_modules_recursive(module + '.' + stem)) return sources +def matches_exclude(subpath: str, exclude: str, fscache: FileSystemCache, verbose: bool) -> bool: + if not exclude: + return False + subpath_str = os.path.abspath(subpath).replace(os.sep, "/") + if fscache.isdir(subpath): + subpath_str += "/" + if re.search(exclude, subpath_str): + if verbose: + print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) + return True + return False + + def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool: """Check that all packages containing id have a __init__ file.""" if path.endswith(('__init__.py', '__init__.pyi')): diff --git a/mypy/options.py b/mypy/options.py index e95ed3e0bb46..752e1cffdb25 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -97,6 +97,8 @@ def __init__(self) -> None: # sufficient to determine module names for files. As a possible alternative, add a single # top-level __init__.py to your packages. self.explicit_package_bases = False + # File names, directory names or subpaths to avoid checking + self.exclude = "" # type: str # disallow_any options self.disallow_any_generics = False diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 5cedec338bbc..056ddf13b108 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -1,8 +1,9 @@ from mypy.modulefinder import BuildSource import os +import pytest import unittest from typing import List, Optional, Set, Tuple -from mypy.find_sources import SourceFinder +from mypy.find_sources import InvalidSourceList, SourceFinder, create_source_list from mypy.fscache import FileSystemCache from mypy.modulefinder import BuildSource from mypy.options import Options @@ -47,10 +48,17 @@ def crawl(finder: SourceFinder, f: str) -> Tuple[str, str]: return module, normalise_path(base_dir) -def find_sources(finder: SourceFinder, f: str) -> List[Tuple[str, Optional[str]]]: +def find_sources_in_dir(finder: SourceFinder, f: str) -> List[Tuple[str, Optional[str]]]: return normalise_build_source_list(finder.find_sources_in_dir(os.path.abspath(f))) +def find_sources( + paths: List[str], options: Options, fscache: FileSystemCache +) -> List[Tuple[str, Optional[str]]]: + paths = [os.path.abspath(p) for p in paths] + return normalise_build_source_list(create_source_list(paths, options, fscache)) + + class SourceFinderSuite(unittest.TestCase): def test_crawl_no_namespace(self) -> None: options = Options() @@ -172,7 +180,7 @@ def test_crawl_namespace_multi_dir(self) -> None: assert crawl(finder, "/a/pkg/a.py") == ("pkg.a", "/a") assert crawl(finder, "/b/pkg/b.py") == ("pkg.b", "/b") - def test_find_sources_no_namespace(self) -> None: + def test_find_sources_in_dir_no_namespace(self) -> None: options = Options() options.namespace_packages = False @@ -184,7 +192,7 @@ def test_find_sources_no_namespace(self) -> None: "/pkg/a2/b/f.py", } finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + assert find_sources_in_dir(finder, "/") == [ ("a2", "/pkg"), ("e", "/pkg/a1/b/c/d"), ("e", "/pkg/a2/b/c/d"), @@ -192,7 +200,7 @@ def test_find_sources_no_namespace(self) -> None: ("f", "/pkg/a2/b"), ] - def test_find_sources_namespace(self) -> None: + def test_find_sources_in_dir_namespace(self) -> None: options = Options() options.namespace_packages = True @@ -204,7 +212,7 @@ def test_find_sources_namespace(self) -> None: "/pkg/a2/b/f.py", } finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + assert find_sources_in_dir(finder, "/") == [ ("a2", "/pkg"), ("a2.b.c.d.e", "/pkg"), ("a2.b.f", "/pkg"), @@ -212,7 +220,7 @@ def test_find_sources_namespace(self) -> None: ("f", "/pkg/a1/b"), ] - def test_find_sources_namespace_explicit_base(self) -> None: + def test_find_sources_in_dir_namespace_explicit_base(self) -> None: options = Options() options.namespace_packages = True options.explicit_package_bases = True @@ -226,7 +234,7 @@ def test_find_sources_namespace_explicit_base(self) -> None: "/pkg/a2/b/f.py", } finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + assert find_sources_in_dir(finder, "/") == [ ("pkg.a1.b.c.d.e", "/"), ("pkg.a1.b.f", "/"), ("pkg.a2", "/"), @@ -236,7 +244,7 @@ def test_find_sources_namespace_explicit_base(self) -> None: options.mypy_path = ["/pkg"] finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + assert find_sources_in_dir(finder, "/") == [ ("a1.b.c.d.e", "/pkg"), ("a1.b.f", "/pkg"), ("a2", "/pkg"), @@ -244,11 +252,112 @@ def test_find_sources_namespace_explicit_base(self) -> None: ("a2.b.f", "/pkg"), ] - def test_find_sources_namespace_multi_dir(self) -> None: + def test_find_sources_in_dir_namespace_multi_dir(self) -> None: options = Options() options.namespace_packages = True options.explicit_package_bases = True options.mypy_path = ["/a", "/b"] finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options) - assert find_sources(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")] + assert find_sources_in_dir(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")] + + def test_find_sources_exclude(self) -> None: + options = Options() + options.namespace_packages = True + + # default + for excluded_dir in ["site-packages", ".whatever", "node_modules", ".x/.z"]: + fscache = FakeFSCache({"/dir/a.py", "/dir/venv/{}/b.py".format(excluded_dir)}) + assert find_sources(["/"], options, fscache) == [("a", "/dir")] + with pytest.raises(InvalidSourceList): + find_sources(["/dir/venv/"], options, fscache) + assert find_sources(["/dir/venv/{}".format(excluded_dir)], options, fscache) == [ + ("b", "/dir/venv/{}".format(excluded_dir)) + ] + assert find_sources(["/dir/venv/{}/b.py".format(excluded_dir)], options, fscache) == [ + ("b", "/dir/venv/{}".format(excluded_dir)) + ] + + files = { + "/pkg/a1/b/c/d/e.py", + "/pkg/a1/b/f.py", + "/pkg/a2/__init__.py", + "/pkg/a2/b/c/d/e.py", + "/pkg/a2/b/f.py", + } + + # file name + options.exclude = r"/f\.py$" + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("e", "/pkg/a1/b/c/d"), + ] + assert find_sources(["/pkg/a1/b/f.py"], options, fscache) == [('f', '/pkg/a1/b')] + assert find_sources(["/pkg/a2/b/f.py"], options, fscache) == [('a2.b.f', '/pkg')] + + # directory name + options.exclude = "/a1/" + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1"], options, fscache) + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1/"], options, fscache) + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1/b"], options, fscache) + + options.exclude = "/a1/$" + assert find_sources(["/pkg/a1"], options, fscache) == [ + ('e', '/pkg/a1/b/c/d'), ('f', '/pkg/a1/b') + ] + + # paths + options.exclude = "/pkg/a1/" + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1"], options, fscache) + + options.exclude = "/(a1|a3)/" + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + + options.exclude = "b/c/" + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ + ("a2", "/pkg"), + ("a2.b.f", "/pkg"), + ("f", "/pkg/a1/b"), + ] + + # nothing should be ignored as a result of this + options.exclude = "|".join(( + "/pkg/a/", "/2", "/1", "/pk/", "/kg", "/g.py", "/bc", "/xxx/pkg/a2/b/f.py" + "xxx/pkg/a2/b/f.py", + )) + fscache = FakeFSCache(files) + assert len(find_sources(["/"], options, fscache)) == len(files) + + files = { + "pkg/a1/b/c/d/e.py", + "pkg/a1/b/f.py", + "pkg/a2/__init__.py", + "pkg/a2/b/c/d/e.py", + "pkg/a2/b/f.py", + } + fscache = FakeFSCache(files) + assert len(find_sources(["/"], options, fscache)) == len(files) diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 2b7ed2b157c5..c974a0248afc 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -19,3 +19,4 @@ pretty = True always_false = MYPYC plugins = misc/proper_plugin.py python_version = 3.5 +exclude = /mypy/typeshed/ diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 8fe9f478a077..4c78928500b0 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -59,7 +59,7 @@ undef undef [out] dir/a.py: error: Duplicate module named 'a' (also at 'dir/subdir/a.py') -dir/a.py: error: Are you missing an __init__.py? +dir/a.py: note: Are you missing an __init__.py? Alternatively, consider using --exclude to avoid checking one of them. == Return code: 2 [case testCmdlineNonPackageSlash] @@ -125,19 +125,7 @@ mypy: can't decode file 'a.py': unknown encoding: uft-8 # type: ignore [out] two/mod/__init__.py: error: Duplicate module named 'mod' (also at 'one/mod/__init__.py') -== Return code: 2 - -[case promptsForgotInit] -# cmd: mypy a.py one/mod/a.py -[file one/__init__.py] -# type: ignore -[file a.py] -# type: ignore -[file one/mod/a.py] -#type: ignore -[out] -one/mod/a.py: error: Duplicate module named 'a' (also at 'a.py') -one/mod/a.py: error: Are you missing an __init__.py? +two/mod/__init__.py: note: Are you missing an __init__.py? Alternatively, consider using --exclude to avoid checking one of them. == Return code: 2 [case testFlagsFile] From 2b52538b9975456e37b237bcf7e67b8d80ea48f2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 10 Feb 2021 17:41:41 +0000 Subject: [PATCH 02/12] Bump version to 0.810 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index 519cd53f74c3..f4ac22846eec 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -5,7 +5,7 @@ # - Release versions have the form "0.NNN". # - Dev versions have the form "0.NNN+dev" (PLUS sign to conform to PEP 440). # - For 1.0 we'll switch back to 1.2.3 form. -__version__ = '0.800' +__version__ = '0.810' base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From 57a2527ce5649eddbd90bca46431a25556a59edb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 11 Feb 2021 11:40:58 +0000 Subject: [PATCH 03/12] Trigger wheel build From 71285ddb8d67ef40277e74617e54f273b574b8dd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 11 Feb 2021 14:43:49 +0000 Subject: [PATCH 04/12] Empty commit to re-trigger builds From 8002fc4cf566929fe931de3b84332a25da286825 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 11 Feb 2021 14:48:06 +0000 Subject: [PATCH 05/12] Move version back to 0.810+dev to trigger builds --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index f4ac22846eec..9e40f614408b 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -5,7 +5,7 @@ # - Release versions have the form "0.NNN". # - Dev versions have the form "0.NNN+dev" (PLUS sign to conform to PEP 440). # - For 1.0 we'll switch back to 1.2.3 form. -__version__ = '0.810' +__version__ = '0.810+dev' base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From e470d93196bb4034d0a4b6914365a82d6c59e35e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 11 Feb 2021 18:23:12 +0000 Subject: [PATCH 06/12] Fix test_find_sources when run under site-packages (#10075) This was failing during wheel builds because we run the tests after installation (under `site-packages`), and this confused the module search logic. Hard code the paths to make this work in any install location. --- mypy/test/test_find_sources.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 056ddf13b108..6d66a28f4e2d 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -1,12 +1,15 @@ -from mypy.modulefinder import BuildSource import os import pytest +import shutil +import tempfile import unittest from typing import List, Optional, Set, Tuple + from mypy.find_sources import InvalidSourceList, SourceFinder, create_source_list from mypy.fscache import FileSystemCache from mypy.modulefinder import BuildSource from mypy.options import Options +from mypy.modulefinder import BuildSource class FakeFSCache(FileSystemCache): @@ -60,6 +63,15 @@ def find_sources( class SourceFinderSuite(unittest.TestCase): + def setUp(self) -> None: + self.tempdir = tempfile.mkdtemp() + self.oldcwd = os.getcwd() + os.chdir(self.tempdir) + + def tearDown(self) -> None: + os.chdir(self.oldcwd) + shutil.rmtree(self.tempdir) + def test_crawl_no_namespace(self) -> None: options = Options() options.namespace_packages = False From 6c5ed189015571b30038df7511834642070ae49c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 11 Feb 2021 19:49:34 +0000 Subject: [PATCH 07/12] Attempt to fix test --- mypy/test/test_find_sources.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 6d66a28f4e2d..d45efa4907c6 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -358,8 +358,8 @@ def test_find_sources_exclude(self) -> None: # nothing should be ignored as a result of this options.exclude = "|".join(( - "/pkg/a/", "/2", "/1", "/pk/", "/kg", "/g.py", "/bc", "/xxx/pkg/a2/b/f.py" - "xxx/pkg/a2/b/f.py", + "/pkg/a/", "/2/", "/1/", "/pk/", "/kg/", r"/g\.py", "/b/d", r"/xxx/pkg/a2/b/f\.py" + r"xxx/pkg/a2/b/f\.py", )) fscache = FakeFSCache(files) assert len(find_sources(["/"], options, fscache)) == len(files) @@ -372,4 +372,5 @@ def test_find_sources_exclude(self) -> None: "pkg/a2/b/f.py", } fscache = FakeFSCache(files) + print('cwd: {}'.format(self.tempdir)) assert len(find_sources(["/"], options, fscache)) == len(files) From 73f18a6a69245b08f92f425a87db99e0992ad28c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Feb 2021 12:37:07 +0000 Subject: [PATCH 08/12] Revert "Attempt to fix test" This reverts commit 6c5ed189015571b30038df7511834642070ae49c. --- mypy/test/test_find_sources.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index d45efa4907c6..6d66a28f4e2d 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -358,8 +358,8 @@ def test_find_sources_exclude(self) -> None: # nothing should be ignored as a result of this options.exclude = "|".join(( - "/pkg/a/", "/2/", "/1/", "/pk/", "/kg/", r"/g\.py", "/b/d", r"/xxx/pkg/a2/b/f\.py" - r"xxx/pkg/a2/b/f\.py", + "/pkg/a/", "/2", "/1", "/pk/", "/kg", "/g.py", "/bc", "/xxx/pkg/a2/b/f.py" + "xxx/pkg/a2/b/f.py", )) fscache = FakeFSCache(files) assert len(find_sources(["/"], options, fscache)) == len(files) @@ -372,5 +372,4 @@ def test_find_sources_exclude(self) -> None: "pkg/a2/b/f.py", } fscache = FakeFSCache(files) - print('cwd: {}'.format(self.tempdir)) assert len(find_sources(["/"], options, fscache)) == len(files) From 28668c8d84bcb1ef4d8222eb2d9fef2589b71feb Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 15 Feb 2021 03:02:53 -0800 Subject: [PATCH 09/12] test_find_sources: check cwd for non root paths (#10077) Co-authored-by: hauntsaninja <> --- mypy/test/test_find_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 6d66a28f4e2d..ba5b613a0948 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -372,4 +372,4 @@ def test_find_sources_exclude(self) -> None: "pkg/a2/b/f.py", } fscache = FakeFSCache(files) - assert len(find_sources(["/"], options, fscache)) == len(files) + assert len(find_sources(["."], options, fscache)) == len(files) From 3b0b05db5057daa1850e55824b43fdefe0d090d6 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 15 Feb 2021 03:04:42 -0800 Subject: [PATCH 10/12] Use relative paths when matching exclude (#10078) Co-authored-by: hauntsaninja <> --- mypy/modulefinder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 2c708b8f802d..82d090702cfc 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -413,7 +413,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: def matches_exclude(subpath: str, exclude: str, fscache: FileSystemCache, verbose: bool) -> bool: if not exclude: return False - subpath_str = os.path.abspath(subpath).replace(os.sep, "/") + subpath_str = os.path.relpath(subpath).replace(os.sep, "/") if fscache.isdir(subpath): subpath_str += "/" if re.search(exclude, subpath_str): From 4373cd5b55dca0ca74269e246efd2be528c8f067 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 16 Feb 2021 13:06:26 +0000 Subject: [PATCH 11/12] Add Python script to build wheels using cibuildwheel (#10096) The contents are extracted from the current GitHub action definition: https://github.com/mypyc/mypy_mypyc-wheels/blob/master/.github/workflows/build.yml This is a fairly direct translation of the existing behavior. The behavior should be identical to the current workflow. The idea is to make this easier to maintain and easier to test locally. This should also make it easier to fix #10074. --- misc/build_wheel.py | 131 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 misc/build_wheel.py diff --git a/misc/build_wheel.py b/misc/build_wheel.py new file mode 100644 index 000000000000..13633405bf77 --- /dev/null +++ b/misc/build_wheel.py @@ -0,0 +1,131 @@ +"""Script to build compiled binary wheels that can be uploaded to PyPI. + +The main GitHub workflow where this script is used: +https://github.com/mypyc/mypy_mypyc-wheels/blob/master/.github/workflows/build.yml + +This uses cibuildwheel (https://github.com/joerick/cibuildwheel) to +build the wheels. + +Usage: + + build_wheel_ci.py --python-version \ + --output-dir + +Wheels for the given Python version will be created in the given directory. +Python version is in form "39". + +This works on macOS, Windows and Linux. + +You can test locally by using --extra-opts. macOS example: + + mypy/misc/build_wheel_ci.py --python-version 39 --output-dir out --extra-opts="--platform macos" + +Other supported values for platform: linux, windows +""" + +import argparse +import os +import subprocess +import sys +from typing import Dict + +# Clang package we use on Linux +LLVM_URL = 'https://github.com/mypyc/mypy_mypyc-wheels/releases/download/llvm/llvm-centos-5.tar.gz' + +# Mypy repository root +ROOT_DIR = os.path.dirname(os.path.dirname(__file__)) + + +def create_environ(python_version: str) -> Dict[str, str]: + """Set up environment variables for cibuildwheel.""" + env = os.environ.copy() + + env['CIBW_BUILD'] = "cp{}-*".format(python_version) + + # Don't build 32-bit wheels + env['CIBW_SKIP'] = "*-manylinux_i686 *-win32" + + env['CIBW_BUILD_VERBOSITY'] = '1' + + # mypy's isolated builds don't specify the requirements mypyc needs, so install + # requirements and don't use isolated builds. we need to use build-requirements.txt + # with recent mypy commits to get stub packages needed for compilation. + # + # TODO: remove use of mypy-requirements.txt once we no longer need to support + # building pre modular typeshed releases + env['CIBW_BEFORE_BUILD'] = """ + pip install -r {package}/mypy-requirements.txt && + (pip install -r {package}/build-requirements.txt || true) + """.replace('\n', ' ') + + # download a copy of clang to use to compile on linux. this was probably built in 2018, + # speeds up compilation 2x + env['CIBW_BEFORE_BUILD_LINUX'] = """ + (cd / && curl -L %s | tar xzf -) && + pip install -r {package}/mypy-requirements.txt && + (pip install -r {package}/build-requirements.txt || true) + """.replace('\n', ' ') % LLVM_URL + + # the double negative is counterintuitive, https://github.com/pypa/pip/issues/5735 + env['CIBW_ENVIRONMENT'] = 'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=3 PIP_NO_BUILD_ISOLATION=no' + env['CIBW_ENVIRONMENT_LINUX'] = ( + 'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=3 PIP_NO_BUILD_ISOLATION=no ' + + 'CC=/opt/llvm/bin/clang' + ) + env['CIBW_ENVIRONMENT_WINDOWS'] = ( + 'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=2 PIP_NO_BUILD_ISOLATION=no' + ) + + # lxml is slow to build wheels for new releases, so allow installing reqs to fail + # if we failed to install lxml, we'll skip tests, but allow the build to succeed + env['CIBW_BEFORE_TEST'] = ( + 'pip install -r {project}/mypy/test-requirements.txt || true' + ) + + # pytest looks for configuration files in the parent directories of where the tests live. + # since we are trying to run the tests from their installed location, we copy those into + # the venv. Ew ew ew. + env['CIBW_TEST_COMMAND'] = """ + ( ! pip list | grep lxml ) || ( + DIR=$(python -c 'import mypy, os; dn = os.path.dirname; print(dn(dn(mypy.__path__[0])))') + && TEST_DIR=$(python -c 'import mypy.test; print(mypy.test.__path__[0])') + && cp '{project}/mypy/pytest.ini' '{project}/mypy/conftest.py' $DIR + && MYPY_TEST_PREFIX='{project}/mypy' pytest $TEST_DIR + ) + """.replace('\n', ' ') + + # i ran into some flaky tests on windows, so only run testcheck. it looks like we + # previously didn't run any tests on windows wheels, so this is a net win. + env['CIBW_TEST_COMMAND_WINDOWS'] = """ + bash -c " + ( ! pip list | grep lxml ) || ( + DIR=$(python -c 'import mypy, os; dn = os.path.dirname; print(dn(dn(mypy.__path__[0])))') + && TEST_DIR=$(python -c 'import mypy.test; print(mypy.test.__path__[0])') + && cp '{project}/mypy/pytest.ini' '{project}/mypy/conftest.py' $DIR + && MYPY_TEST_PREFIX='{project}/mypy' pytest $TEST_DIR/testcheck.py + ) + " + """.replace('\n', ' ') + return env + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--python-version', required=True, metavar='XY', + help='Python version (e.g. 38 or 39)') + parser.add_argument('--output-dir', required=True, metavar='DIR', + help='Output directory for created wheels') + parser.add_argument('--extra-opts', default='', metavar='OPTIONS', + help='Extra options passed to cibuildwheel verbatim') + args = parser.parse_args() + python_version = args.python_version + output_dir = args.output_dir + extra_opts = args.extra_opts + environ = create_environ(python_version) + script = 'python -m cibuildwheel {} --output-dir {} {}'.format(extra_opts, output_dir, + ROOT_DIR) + subprocess.check_call(script, shell=True, env=environ) + + +if __name__ == '__main__': + main() From d089891198ef470c8bec9bd7d7b50a02757c5b68 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 19 Feb 2021 16:17:29 +0000 Subject: [PATCH 12/12] Bump version to 0.812 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index 9e40f614408b..2e869f8a511e 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -5,7 +5,7 @@ # - Release versions have the form "0.NNN". # - Dev versions have the form "0.NNN+dev" (PLUS sign to conform to PEP 440). # - For 1.0 we'll switch back to 1.2.3 form. -__version__ = '0.810+dev' +__version__ = '0.812' base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))