Skip to content

[BE] add a linter to check consistency for cmake minimum version in requirements #156961

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 24 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .lintrunner.toml
Original file line number Diff line number Diff line change
Expand Up @@ -1487,6 +1487,31 @@ init_command = [
'tomli==2.2.1 ; python_version < "3.11"',
]

[[linter]]
code = 'CMAKE_MINIMUM_REQUIRED'
command = [
'python3',
'tools/linter/adapters/cmake_minimum_required_linter.py',
'--',
'@{{PATHSFILE}}'
]
include_patterns = [
"**/pyproject.toml",
"**/CMakeLists.txt",
"**/CMakeLists.txt.in",
"**/*.cmake",
"**/*.cmake.in",
"**/*requirements*.txt",
"**/*requirements*.in",
]
init_command = [
'python3',
'tools/linter/adapters/pip_init.py',
'--dry-run={{DRYRUN}}',
'packaging==25.0',
'tomli==2.2.1 ; python_version < "3.11"',
]

[[linter]]
code = 'COPYRIGHT'
include_patterns = ['**']
Expand Down
245 changes: 245 additions & 0 deletions tools/linter/adapters/cmake_minimum_required_linter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,245 @@
from __future__ import annotations

import argparse
import concurrent.futures
import fnmatch
import json
import logging
import os
import re
import sys
from enum import Enum
from pathlib import Path
from typing import NamedTuple

from packaging.requirements import Requirement
from packaging.version import Version


if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib # type: ignore[import-not-found]


REPO_ROOT = Path(__file__).absolute().parents[3]
sys.path.insert(0, str(REPO_ROOT))

from tools.setup_helpers.env import CMAKE_MINIMUM_VERSION_STRING


sys.path.remove(str(REPO_ROOT))


LINTER_CODE = "CMAKE_MINIMUM_REQUIRED"
CMAKE_MINIMUM_VERSION = Version(CMAKE_MINIMUM_VERSION_STRING)


class LintSeverity(str, Enum):
ERROR = "error"
WARNING = "warning"
ADVICE = "advice"
DISABLED = "disabled"


class LintMessage(NamedTuple):
path: str | None
line: int | None
char: int | None
code: str
severity: LintSeverity
name: str
original: str | None
replacement: str | None
description: str | None


def format_error_message(
filename: str,
error: Exception | None = None,
*,
line: int | None = None,
message: str | None = None,
) -> LintMessage:
if message is None and error is not None:
message = f"Failed due to {error.__class__.__name__}:\n{error}"
return LintMessage(
path=filename,
line=line,
char=None,
code=LINTER_CODE,
severity=LintSeverity.ERROR,
name="CMake minimum version",
original=None,
replacement=None,
description=message,
)


CMAKE_MINIMUM_REQUIRED_PATTERN = re.compile(
r"cmake_minimum_required\(VERSION\s+(?P<version>\d+\.\d+(\.\d+)?)\b.*\)",
flags=re.IGNORECASE,
)


def check_cmake(path: Path) -> list[LintMessage]:
with path.open(encoding="utf-8") as f:
for i, line in enumerate(f, start=1):
if match := CMAKE_MINIMUM_REQUIRED_PATTERN.search(line):
version = match.group("version")
if path.samefile(REPO_ROOT / "CMakeLists.txt"):
if Version(version) != CMAKE_MINIMUM_VERSION:
return [
format_error_message(
str(path),
line=i,
message=(
f"CMake minimum version must be {CMAKE_MINIMUM_VERSION}, "
f"but found {version}."
),
)
]
elif Version(version) > CMAKE_MINIMUM_VERSION:
return [
format_error_message(
str(path),
line=i,
message=(
f"The environment can only provide CMake {CMAKE_MINIMUM_VERSION}, "
f"but found requiring {version}."
),
)
]
return []


def check_requirement(
requirement: Requirement,
path: Path,
*,
line: int | None = None,
) -> LintMessage | None:
if requirement.name.lower() != "cmake":
return None

for spec in requirement.specifier:
if (
spec.operator in ("==", ">=")
and Version(spec.version.removesuffix(".*")) < CMAKE_MINIMUM_VERSION
):
return format_error_message(
str(path),
line=line,
message=(
f"CMake minimum version must be at least {CMAKE_MINIMUM_VERSION}, "
f"but found {spec}."
),
)

return None


def check_pyproject(path: Path) -> list[LintMessage]:
try:
pyproject = tomllib.loads(path.read_text(encoding="utf-8"))
except (tomllib.TOMLDecodeError, OSError) as err:
return [format_error_message(str(path), err)]

if not isinstance(pyproject, dict):
return []
if not isinstance(pyproject.get("build-system"), dict):
return []

build_system = pyproject["build-system"]
requires = build_system.get("requires")
if not isinstance(requires, list):
return []
return list(
filter(
None,
(check_requirement(Requirement(req), path=path) for req in requires),
)
)


def check_requirements(path: Path) -> list[LintMessage]:
try:
with path.open(encoding="utf-8") as f:
lines = f.readlines()
except OSError as err:
return [format_error_message(str(path), err)]

lint_messages = []
for i, line in enumerate(lines, start=1):
line = line.strip()
if not line or line.startswith(("#", "-")):
continue
try:
requirement = Requirement(line)
except Exception:
continue
lint_message = check_requirement(requirement, path=path, line=i)
if lint_message is not None:
lint_messages.append(lint_message)

return lint_messages


def check_file(filename: str) -> list[LintMessage]:
path = Path(filename).absolute()
basename = path.name.lower()
if basename in ("cmakelists.txt", "cmakelists.txt.in") or basename.endswith(
(".cmake", ".cmake.in")
):
return check_cmake(path)
if basename == "pyproject.toml":
return check_pyproject(path)
if fnmatch.fnmatch(basename, "*requirements*.txt") or fnmatch.fnmatch(
basename, "*requirements*.in"
):
return check_requirements(path)
return []


def main() -> None:
parser = argparse.ArgumentParser(
description="Check consistency of cmake minimum version in requirement files.",
fromfile_prefix_chars="@",
)
parser.add_argument(
"--verbose",
action="store_true",
help="verbose logging",
)
parser.add_argument(
"filenames",
nargs="+",
help="paths to lint",
)
args = parser.parse_args()

logging.basicConfig(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: attach logging config to specific logger instead.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is copied from other files.

format="<%(processName)s:%(levelname)s> %(message)s",
level=logging.NOTSET
if args.verbose
else logging.DEBUG
if len(args.filenames) < 1000
else logging.INFO,
stream=sys.stderr,
)

with concurrent.futures.ProcessPoolExecutor(
max_workers=os.cpu_count(),
) as executor:
futures = {executor.submit(check_file, x): x for x in args.filenames}
for future in concurrent.futures.as_completed(futures):
try:
for lint_message in future.result():
print(json.dumps(lint_message._asdict()), flush=True)
except Exception:
logging.critical('Failed at "%s".', futures[future])
raise


if __name__ == "__main__":
main()
53 changes: 29 additions & 24 deletions tools/setup_helpers/cmake.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,14 @@
from typing import cast

from .cmake_utils import CMakeValue, get_cmake_cache_variables_from_file
from .env import BUILD_DIR, check_negative_env_flag, IS_64BIT, IS_DARWIN, IS_WINDOWS
from .env import (
BUILD_DIR,
check_negative_env_flag,
CMAKE_MINIMUM_VERSION_STRING,
IS_64BIT,
IS_DARWIN,
IS_WINDOWS,
)


try:
Expand Down Expand Up @@ -50,6 +57,9 @@ def _mkdir_p(d: str) -> None:
USE_NINJA = os.environ["CMAKE_GENERATOR"].lower() == "ninja"


CMAKE_MINIMUM_VERSION = Version(CMAKE_MINIMUM_VERSION_STRING)


class CMake:
"Manages cmake."

Expand Down Expand Up @@ -79,32 +89,27 @@ def _ninja_build_file(self) -> str:
def _get_cmake_command() -> str:
"""Returns cmake command."""

cmake_command = "cmake"
if IS_WINDOWS:
return cmake_command
cmake3_version = CMake._get_version(shutil.which("cmake3"))
cmake_version = CMake._get_version(shutil.which("cmake"))

_cmake_min_version = Version("3.27.0")
if all(
ver is None or ver < _cmake_min_version
for ver in [cmake_version, cmake3_version]
):
return "cmake"

cmake_versions: dict[str, Version] = {}
for cmd in ("cmake", "cmake3"):
command = shutil.which(cmd)
ver = CMake._get_version(command)
if ver is not None:
eprint(f"Found {cmd} ({command}) version: {ver}", end="")
if ver >= CMAKE_MINIMUM_VERSION:
eprint(f" (>={CMAKE_MINIMUM_VERSION})")
cmake_versions[cmd] = ver
else:
eprint(f" (<{CMAKE_MINIMUM_VERSION})")

if not cmake_versions:
raise RuntimeError(
"no cmake or cmake3 with version >= 3.27.0 found:"
+ str([cmake_version, cmake3_version])
f"no cmake or cmake3 with version >= {CMAKE_MINIMUM_VERSION}, "
f"found: {list(cmake_versions.values())}"
)

if cmake3_version is None:
cmake_command = "cmake"
elif cmake_version is None:
cmake_command = "cmake3"
else:
if cmake3_version >= cmake_version:
cmake_command = "cmake3"
else:
cmake_command = "cmake"
return cmake_command
return max(cmake_versions, key=cmake_versions.get) # type: ignore[arg-type]

@staticmethod
def _get_version(cmd: str | None) -> Version | None:
Expand Down
2 changes: 2 additions & 0 deletions tools/setup_helpers/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from collections.abc import Iterable


CMAKE_MINIMUM_VERSION_STRING = "3.27"

IS_WINDOWS = platform.system() == "Windows"
IS_DARWIN = platform.system() == "Darwin"
IS_LINUX = platform.system() == "Linux"
Expand Down
Loading