Skip to content

Commit 6f93fc1

Browse files
evemorgenevemorgen
authored
Tests autodiscovery replacing hardcoded file list (#13143)
This PR hopefully resolves #8650. The main change is the `find_test_files ` helper function that looks for files matching the pattern in the unit tests directory. It has been used in the following test files for autodiscovery: - `testcheck.py` - `testdeps.py` - ` testfinegrained.py` - `testparse.py` - `testsemanal.py` I've also updated the readme.
1 parent b6d525d commit 6f93fc1

File tree

7 files changed

+47
-134
lines changed

7 files changed

+47
-134
lines changed

‎mypy/test/helpers.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
import os
2+
import pathlib
23
import re
34
import sys
45
import time
56
import shutil
67
import contextlib
78

8-
from typing import List, Iterable, Dict, Tuple, Callable, Any, Iterator, Union, Pattern
9+
from typing import List, Iterable, Dict, Tuple, Callable, Any, Iterator, Union, Pattern, Optional
910

1011
from mypy import defaults
1112
import mypy.api as api
@@ -494,3 +495,11 @@ def normalize_file_output(content: List[str], current_abs_path: str) -> List[str
494495
result = [re.sub(r'\b' + re.escape(base_version) + r'\b', '$VERSION', x) for x in result]
495496
result = [timestamp_regex.sub('$TIMESTAMP', x) for x in result]
496497
return result
498+
499+
500+
def find_test_files(pattern: str, exclude: Optional[List[str]] = None) -> List[str]:
501+
return [
502+
path.name
503+
for path in (pathlib.Path("./test-data/unit").rglob(pattern))
504+
if path.name not in (exclude or [])
505+
]

‎mypy/test/testcheck.py

Lines changed: 11 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
assert_string_arrays_equal, normalize_error_messages, assert_module_equivalence,
1818
update_testcase_output, parse_options,
1919
assert_target_equivalence, check_test_output_files, perform_file_operations,
20+
find_test_files,
2021
)
2122
from mypy.errors import CompileError
2223
from mypy.semanal_main import core_modules
@@ -29,96 +30,20 @@
2930
import pytest
3031

3132
# List of files that contain test case descriptions.
32-
typecheck_files = [
33-
'check-basic.test',
34-
'check-union-or-syntax.test',
35-
'check-callable.test',
36-
'check-classes.test',
37-
'check-statements.test',
38-
'check-generics.test',
39-
'check-dynamic-typing.test',
40-
'check-inference.test',
41-
'check-inference-context.test',
42-
'check-kwargs.test',
43-
'check-overloading.test',
44-
'check-type-checks.test',
45-
'check-abstract.test',
46-
'check-multiple-inheritance.test',
47-
'check-super.test',
48-
'check-modules.test',
49-
'check-modules-fast.test',
50-
'check-typevar-values.test',
51-
'check-unsupported.test',
52-
'check-unreachable-code.test',
53-
'check-unions.test',
54-
'check-isinstance.test',
55-
'check-lists.test',
56-
'check-namedtuple.test',
57-
'check-narrowing.test',
58-
'check-typeddict.test',
59-
'check-type-aliases.test',
60-
'check-ignore.test',
61-
'check-type-promotion.test',
62-
'check-semanal-error.test',
63-
'check-flags.test',
64-
'check-incremental.test',
65-
'check-serialize.test',
66-
'check-bound.test',
67-
'check-optional.test',
68-
'check-fastparse.test',
69-
'check-warnings.test',
70-
'check-async-await.test',
71-
'check-newtype.test',
72-
'check-class-namedtuple.test',
73-
'check-selftype.test',
74-
'check-python2.test',
75-
'check-columns.test',
76-
'check-functions.test',
77-
'check-tuples.test',
78-
'check-expressions.test',
79-
'check-generic-subtyping.test',
80-
'check-varargs.test',
81-
'check-newsyntax.test',
82-
'check-protocols.test',
83-
'check-underscores.test',
84-
'check-classvar.test',
85-
'check-enum.test',
86-
'check-incomplete-fixture.test',
87-
'check-custom-plugin.test',
88-
'check-default-plugin.test',
89-
'check-attr.test',
90-
'check-ctypes.test',
91-
'check-dataclasses.test',
92-
'check-final.test',
93-
'check-redefine.test',
94-
'check-literal.test',
95-
'check-newsemanal.test',
96-
'check-inline-config.test',
97-
'check-reports.test',
98-
'check-errorcodes.test',
99-
'check-annotated.test',
100-
'check-parameter-specification.test',
101-
'check-typevar-tuple.test',
102-
'check-generic-alias.test',
103-
'check-typeguard.test',
104-
'check-functools.test',
105-
'check-singledispatch.test',
106-
'check-slots.test',
107-
'check-formatting.test',
108-
'check-native-int.test',
109-
]
33+
# Includes all check-* files with the .test extension in the test-data/unit directory
34+
typecheck_files = find_test_files(pattern="check-*.test")
11035

11136
# Tests that use Python 3.8-only AST features (like expression-scoped ignores):
112-
if sys.version_info >= (3, 8):
113-
typecheck_files.append('check-python38.test')
114-
if sys.version_info >= (3, 9):
115-
typecheck_files.append('check-python39.test')
116-
if sys.version_info >= (3, 10):
117-
typecheck_files.append('check-python310.test')
37+
if sys.version_info < (3, 8):
38+
typecheck_files.remove('check-python38.test')
39+
if sys.version_info < (3, 9):
40+
typecheck_files.remove('check-python39.test')
41+
if sys.version_info < (3, 10):
42+
typecheck_files.remove('check-python310.test')
11843

11944
# Special tests for platforms with case-insensitive filesystems.
120-
if sys.platform in ('darwin', 'win32'):
121-
typecheck_files.extend(['check-modules-case.test'])
45+
if sys.platform not in ('darwin', 'win32'):
46+
typecheck_files.remove('check-modules-case.test')
12247

12348

12449
class TypeCheckSuite(DataSuite):

‎mypy/test/testdeps.py

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from mypy.server.deps import get_dependencies
1515
from mypy.test.config import test_temp_dir
1616
from mypy.test.data import DataDrivenTestCase, DataSuite
17-
from mypy.test.helpers import assert_string_arrays_equal, parse_options
17+
from mypy.test.helpers import assert_string_arrays_equal, parse_options, find_test_files
1818
from mypy.types import Type
1919
from mypy.typestate import TypeState
2020

@@ -23,14 +23,7 @@
2323

2424

2525
class GetDependenciesSuite(DataSuite):
26-
files = [
27-
'deps.test',
28-
'deps-types.test',
29-
'deps-generics.test',
30-
'deps-expressions.test',
31-
'deps-statements.test',
32-
'deps-classes.test',
33-
]
26+
files = find_test_files(pattern="deps*.test")
3427

3528
def run_case(self, testcase: DataDrivenTestCase) -> None:
3629
src = '\n'.join(testcase.input)

‎mypy/test/testfinegrained.py

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
)
2828
from mypy.test.helpers import (
2929
assert_string_arrays_equal, parse_options, assert_module_equivalence,
30-
assert_target_equivalence, perform_file_operations,
30+
assert_target_equivalence, perform_file_operations, find_test_files,
3131
)
3232
from mypy.server.mergecheck import check_consistency
3333
from mypy.dmypy_util import DEFAULT_STATUS_FILE
@@ -42,15 +42,9 @@
4242

4343

4444
class FineGrainedSuite(DataSuite):
45-
files = [
46-
'fine-grained.test',
47-
'fine-grained-cycles.test',
48-
'fine-grained-blockers.test',
49-
'fine-grained-modules.test',
50-
'fine-grained-follow-imports.test',
51-
'fine-grained-suggest.test',
52-
'fine-grained-attr.test',
53-
]
45+
files = find_test_files(
46+
pattern="fine-grained*.test", exclude=["fine-grained-cache-incremental.test"]
47+
)
5448

5549
# Whether to use the fine-grained cache in the testing. This is overridden
5650
# by a trivial subclass to produce a suite that uses the cache.

‎mypy/test/testparse.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from pytest import skip
66

77
from mypy import defaults
8-
from mypy.test.helpers import assert_string_arrays_equal, parse_options
8+
from mypy.test.helpers import assert_string_arrays_equal, parse_options, find_test_files
99
from mypy.test.data import DataDrivenTestCase, DataSuite
1010
from mypy.parse import parse
1111
from mypy.errors import CompileError
@@ -15,11 +15,10 @@
1515
class ParserSuite(DataSuite):
1616
required_out_section = True
1717
base_path = '.'
18-
files = ['parse.test',
19-
'parse-python2.test']
18+
files = find_test_files(pattern="parse*.test", exclude=["parse-errors.test"])
2019

21-
if sys.version_info >= (3, 10):
22-
files.append('parse-python310.test')
20+
if sys.version_info < (3, 10):
21+
files.remove('parse-python310.test')
2322

2423
def run_case(self, testcase: DataDrivenTestCase) -> None:
2524
test_parser(testcase)

‎mypy/test/testsemanal.py

Lines changed: 15 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
from mypy.modulefinder import BuildSource
1010
from mypy.defaults import PYTHON3_VERSION
1111
from mypy.test.helpers import (
12-
assert_string_arrays_equal, normalize_error_messages, testfile_pyversion, parse_options
12+
assert_string_arrays_equal, normalize_error_messages, testfile_pyversion, parse_options,
13+
find_test_files,
1314
)
1415
from mypy.test.data import DataDrivenTestCase, DataSuite
1516
from mypy.test.config import test_temp_dir
@@ -21,26 +22,19 @@
2122
# Semantic analyzer test cases: dump parse tree
2223

2324
# Semantic analysis test case description files.
24-
semanal_files = [
25-
'semanal-basic.test',
26-
'semanal-expressions.test',
27-
'semanal-classes.test',
28-
'semanal-types.test',
29-
'semanal-typealiases.test',
30-
'semanal-modules.test',
31-
'semanal-statements.test',
32-
'semanal-abstractclasses.test',
33-
'semanal-namedtuple.test',
34-
'semanal-typeddict.test',
35-
'semenal-literal.test',
36-
'semanal-classvar.test',
37-
'semanal-python2.test',
38-
'semanal-lambda.test',
39-
]
40-
41-
42-
if sys.version_info >= (3, 10):
43-
semanal_files.append('semanal-python310.test')
25+
semanal_files = find_test_files(
26+
pattern="semanal-*.test",
27+
exclude=[
28+
"semanal-errors-python310.test",
29+
"semanal-errors.test",
30+
"semanal-typeinfo.test",
31+
"semanal-symtable.test",
32+
],
33+
)
34+
35+
36+
if sys.version_info < (3, 10):
37+
semanal_files.remove('semanal-python310.test')
4438

4539

4640
def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Options:

‎test-data/unit/README.md

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ Quick Start
77

88
To add a simple unit test for a new feature you developed, open or create a
99
`test-data/unit/check-*.test` file with a name that roughly relates to the
10-
feature you added. If you added a new `check-*.test` file, add it to the list
11-
of files in `mypy/test/testcheck.py`.
10+
feature you added. If you added a new `check-*.test` file, it will be autodiscovered during unittests run.
1211

1312
Add the test in this format anywhere in the file:
1413

0 commit comments

Comments
 (0)