From 8f26df4185e7a83a8f7c194ffdfadeb89842823e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 16 Aug 2025 09:54:11 -0700 Subject: [PATCH] conformance tests: drop timing information Timing information is nice to have but the current data is not very reliable, as it is the timing on whatever machine was used by the person who most recently regenerated the results. It's also a big source of merge conflicts, basically meaning that every pair of changes to the conformance suite will merge conflict with each other. To make it more pleasant to work with the conformance suite, let's just drop the timings. --- conformance/results/mypy/version.toml | 1 - conformance/results/pyright/version.toml | 1 - conformance/results/results.html | 4 ---- conformance/results/zuban/version.toml | 1 - conformance/src/main.py | 13 +++++-------- conformance/src/options.py | 6 ------ conformance/src/reporting.py | 7 ------- 7 files changed, 5 insertions(+), 28 deletions(-) diff --git a/conformance/results/mypy/version.toml b/conformance/results/mypy/version.toml index 867a79fe2..b72d4180d 100644 --- a/conformance/results/mypy/version.toml +++ b/conformance/results/mypy/version.toml @@ -1,2 +1 @@ version = "mypy 1.17.1" -test_duration = 2.04 diff --git a/conformance/results/pyright/version.toml b/conformance/results/pyright/version.toml index 7e373c1a8..ec4415b18 100644 --- a/conformance/results/pyright/version.toml +++ b/conformance/results/pyright/version.toml @@ -1,2 +1 @@ version = "pyright 1.1.403" -test_duration = 1.83 diff --git a/conformance/results/results.html b/conformance/results/results.html index cce50c1e0..bef4333bc 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -159,16 +159,12 @@

Python Type System Conformance Test Results

") summary_html.append("")
 
mypy 1.17.1
-
2.0sec
pyright 1.1.403
-
1.8sec
pyre 0.9.25
-
10.3sec
zuban 0.0.19
-
0.35sec
diff --git a/conformance/results/zuban/version.toml b/conformance/results/zuban/version.toml index 1f4c294b7..9f28f9ef3 100644 --- a/conformance/results/zuban/version.toml +++ b/conformance/results/zuban/version.toml @@ -1,2 +1 @@ version = "zuban 0.0.19" -test_duration = 0.35 diff --git a/conformance/src/main.py b/conformance/src/main.py index 484688b77..4b8a683dd 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -22,7 +22,6 @@ def run_tests( root_dir: Path, type_checker: TypeChecker, test_cases: Sequence[Path], - skip_timing: bool = False, ): print(f"Running tests for {type_checker.name}") @@ -30,6 +29,8 @@ def run_tests( tests_output = type_checker.run_tests([file.name for file in test_cases]) test_duration = time() - test_start_time + print(f"Completed tests for {type_checker.name} in {test_duration:.2f} seconds") + for _, output in tests_output.items(): type_checker.parse_errors(output.splitlines()) @@ -40,7 +41,7 @@ def run_tests( type_checker, results_dir, test_case, tests_output.get(test_case.name, "") ) - update_type_checker_info(type_checker, root_dir, test_duration, skip_timing=skip_timing) + update_type_checker_info(type_checker, root_dir) def get_expected_errors(test_case: Path) -> tuple[ @@ -217,9 +218,7 @@ def update_output_for_test( tomlkit.dump(existing_results, f) -def update_type_checker_info( - type_checker: TypeChecker, root_dir: Path, test_duration: float, skip_timing: bool = False -): +def update_type_checker_info(type_checker: TypeChecker, root_dir: Path): # Record the version of the type checker used for the latest run. version_file = root_dir / "results" / type_checker.name / "version.toml" @@ -234,8 +233,6 @@ def update_type_checker_info( existing_info = {} existing_info["version"] = type_checker.get_version() - if not skip_timing: - existing_info["test_duration"] = round(test_duration, 2) version_file.parent.mkdir(parents=True, exist_ok=True) with open(version_file, "w") as f: @@ -266,7 +263,7 @@ def main(): if not type_checker.install(): print(f"Skipping tests for {type_checker.name}") else: - run_tests(root_dir, type_checker, test_cases, skip_timing=options.skip_timing) + run_tests(root_dir, type_checker, test_cases) # Generate a summary report. generate_summary(root_dir) diff --git a/conformance/src/options.py b/conformance/src/options.py index 93caef022..89120774e 100644 --- a/conformance/src/options.py +++ b/conformance/src/options.py @@ -9,7 +9,6 @@ @dataclass class _Options: report_only: bool | None - skip_timing: bool def parse_options(argv: list[str]) -> _Options: @@ -20,10 +19,5 @@ def parse_options(argv: list[str]) -> _Options: action="store_true", help="regenerates the test suite report from past results", ) - reporting_group.add_argument( - "--skip-timing", - action="store_true", - help="do not update timing information in the output files", - ) ret = _Options(**vars(parser.parse_args(argv))) return ret diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index c17687240..cff2dc465 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -46,15 +46,8 @@ def generate_summary_html(root_dir: Path) -> str: existing_info = {} version = existing_info["version"] or "Unknown version" - test_duration = existing_info.get("test_duration") summary_html.append(f"
{version}
") - if test_duration is not None: - if test_duration < 1: - duration = f"{test_duration:.2f}" - else: - duration = f"{test_duration:.1f}" - summary_html.append(f"
{duration}sec
") summary_html.append("